@opentermsarchive/engine 0.26.0 → 0.27.0

package/scripts/dataset/export/index.test.js

@@ -8,8 +8,8 @@ import dircompare from 'dir-compare';
 import mime from 'mime';
 import StreamZip from 'node-stream-zip';
 
-import Record from '../../../src/archivist/recorder/record.js';
 import GitRepository from '../../../src/archivist/recorder/repositories/git/index.js';
+import Version from '../../../src/archivist/recorder/version.js';
 
 import generateArchive from './index.js';
 
@@ -20,8 +20,8 @@ const { expect } = chai;
 const FIRST_SERVICE_PROVIDER_ID = 'ServiceA';
 const SECOND_SERVICE_PROVIDER_ID = 'ServiceB';
 
-const FIRST_DOCUMENT_TYPE = 'Terms of Service';
-const SECOND_DOCUMENT_TYPE = 'Privacy Policy';
+const FIRST_TERMS_TYPE = 'Terms of Service';
+const SECOND_TERMS_TYPE = 'Privacy Policy';
 
 const FIRST_FETCH_DATE = '2021-01-01T11:27:00.000Z';
 const SECOND_FETCH_DATE = '2021-01-11T11:32:47.000Z';
@@ -31,8 +31,6 @@ const FOURTH_FETCH_DATE = '2022-01-01T12:12:24.000Z';
 const FIRST_CONTENT = 'First Content';
 const SECOND_CONTENT = 'Second Content';
 
-const MIME_TYPE = 'text/markdown';
-
 const SNAPSHOT_ID = '721ce4a63ad399ecbdb548a66d6d327e7bc97876';
 
 const RELEASE_DATE = '2022-01-01T18:21:00.000Z';
@@ -56,38 +54,34 @@ describe('Export', () => {
 
       await repository.initialize();
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: FIRST_SERVICE_PROVIDER_ID,
-        documentType: FIRST_DOCUMENT_TYPE,
+        termsType: FIRST_TERMS_TYPE,
         content: FIRST_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: FIRST_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: FIRST_SERVICE_PROVIDER_ID,
-        documentType: FIRST_DOCUMENT_TYPE,
+        termsType: FIRST_TERMS_TYPE,
         content: SECOND_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: SECOND_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: SECOND_SERVICE_PROVIDER_ID,
-        documentType: FIRST_DOCUMENT_TYPE,
+        termsType: FIRST_TERMS_TYPE,
         content: FIRST_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: THIRD_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));
 
-      await repository.save(new Record({
+      await repository.save(new Version({
         serviceId: SECOND_SERVICE_PROVIDER_ID,
-        documentType: SECOND_DOCUMENT_TYPE,
+        termsType: SECOND_TERMS_TYPE,
         content: FIRST_CONTENT,
-        mimeType: MIME_TYPE,
         fetchDate: FOURTH_FETCH_DATE,
         snapshotId: SNAPSHOT_ID,
       }));

package/scripts/dataset/export/test/fixtures/dataset/README.md

@@ -2,7 +2,7 @@
 
 This dataset consolidates the contractual documents of 2 service providers, in all their versions that were accessible online between January 1, 2021 and January 6, 2022.
 
-This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [https://github.com/OpenTermsArchive/sandbox](https://github.com/OpenTermsArchive/sandbox).
+This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [https://github.com/OpenTermsArchive/sandbox-versions](https://github.com/OpenTermsArchive/sandbox-versions).
 
 It has been generated with [Open Terms Archive](https://opentermsarchive.org).
 

package/scripts/declarations/lint/index.mocha.js

@@ -31,7 +31,7 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate } = await declarationUtils.getModifiedServiceDocumentTypes());
+    ({ services: servicesToValidate } = await declarationUtils.getModifiedServiceTermsTypes());
   }
 
   const lintFile = lintAndFixFile(options.fix);

package/scripts/declarations/utils/index.js

@@ -24,27 +24,27 @@ export default class DeclarationUtils {
 
     const modifiedFilePaths = modifiedFilePathsAsString ? modifiedFilePathsAsString.split('\n') : [];
 
-    return { modifiedFilePaths, modifiedServiceIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.filePathToServiceId))) };
+    return { modifiedFilePaths, modifiedServicesIds: Array.from(new Set(modifiedFilePaths.map(DeclarationUtils.filePathToServiceId))) };
   }
 
   async getModifiedServices() {
-    const { modifiedServiceIds } = await this.getModifiedData();
+    const { modifiedServicesIds } = await this.getModifiedData();
 
-    return modifiedServiceIds;
+    return modifiedServicesIds;
   }
 
-  async getModifiedServiceDocumentTypes() {
-    const { modifiedFilePaths, modifiedServiceIds } = await this.getModifiedData();
-    const servicesDocumentTypes = {};
+  async getModifiedServiceTermsTypes() {
+    const { modifiedFilePaths, modifiedServicesIds } = await this.getModifiedData();
+    const servicesTermsTypes = {};
 
     await Promise.all(modifiedFilePaths.map(async modifiedFilePath => {
       const serviceId = DeclarationUtils.filePathToServiceId(modifiedFilePath);
 
       if (!modifiedFilePath.endsWith('.json')) {
         // Here we should compare AST of both files to detect on which function
-        // change has been made, and then find which document type depends on this
+        // change has been made, and then find which terms type depends on this
         // function.
-        // As this is a complicated process, we will just send back all document types
+        // As this is a complicated process, we will just send back all terms types
         const declaration = await this.getJSONFile(`declarations/${serviceId}.json`, this.defaultBranch);
 
         return Object.keys(declaration.documents);
@@ -60,7 +60,7 @@ export default class DeclarationUtils {
         return;
       }
 
-      const modifiedDocumentTypes = diff.reduce((acc, { path }) => {
+      const modifiedTermsTypes = diff.reduce((acc, { path }) => {
         if (modifiedFilePath.includes('.history')) {
           acc.add(path[0]);
         } else if (path[0] == 'documents') {
@@ -70,12 +70,12 @@ export default class DeclarationUtils {
         return acc;
       }, new Set());
 
-      servicesDocumentTypes[serviceId] = Array.from(new Set([ ...servicesDocumentTypes[serviceId] || [], ...modifiedDocumentTypes ]));
+      servicesTermsTypes[serviceId] = Array.from(new Set([ ...servicesTermsTypes[serviceId] || [], ...modifiedTermsTypes ]));
     }));
 
     return {
-      services: modifiedServiceIds,
-      servicesDocumentTypes,
+      services: modifiedServicesIds,
+      servicesTermsTypes,
     };
   }
 }

package/scripts/declarations/validate/definitions.js

@@ -25,7 +25,7 @@ const definitions = {
     ],
   },
   contentSelectors: { $ref: '#/definitions/selectors' },
-  noiseSelectors: { $ref: '#/definitions/selectors' },
+  insignificantContentSelectors: { $ref: '#/definitions/selectors' },
   filters: {
     type: 'array',
     items: {

package/scripts/declarations/validate/index.mocha.js

@@ -6,8 +6,8 @@ import { expect } from 'chai';
 import config from 'config';
 import jsonSourceMap from 'json-source-map';
 
+import extract from '../../../src/archivist/extract/index.js';
 import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from '../../../src/archivist/fetcher/index.js';
-import filter from '../../../src/archivist/filter/index.js';
 import * as services from '../../../src/archivist/services/index.js';
 import DeclarationUtils from '../utils/index.js';
 
@@ -25,8 +25,8 @@ const instancePath = path.resolve(declarationsPath, '../');
 export default async options => {
   const schemaOnly = options.schemaOnly || false;
   let servicesToValidate = options.services || [];
-  const documentTypes = options.termsTypes || [];
-  let servicesDocumentTypes = {};
+  const termsTypes = options.types || [];
+  let servicesTermsTypes = {};
 
   const serviceDeclarations = await services.loadWithHistory(servicesToValidate);
 
@@ -37,7 +37,7 @@ export default async options => {
   if (options.modified) {
     const declarationUtils = new DeclarationUtils(instancePath);
 
-    ({ services: servicesToValidate, servicesDocumentTypes } = await declarationUtils.getModifiedServiceDocumentTypes());
+    ({ services: servicesToValidate, servicesTermsTypes } = await declarationUtils.getModifiedServiceTermsTypes());
   }
 
   describe('Service declarations validation', async function () {
@@ -76,61 +76,57 @@ export default async options => {
         }
 
         if (!schemaOnly && service) {
-          service.getDocumentTypes()
-            .filter(documentType => {
-              if (servicesDocumentTypes[serviceId] && servicesDocumentTypes[serviceId].length > 0) {
-                return servicesDocumentTypes[serviceId].includes(documentType);
+          service.getTermsTypes()
+            .filter(termsType => {
+              if (servicesTermsTypes[serviceId] && servicesTermsTypes[serviceId].length > 0) {
+                return servicesTermsTypes[serviceId].includes(termsType);
               }
 
-              if (documentTypes.length > 0) {
-                return documentTypes.includes(documentType);
+              if (termsTypes.length > 0) {
+                return termsTypes.includes(termsType);
               }
 
               return true;
             })
             .forEach(type => {
               describe(type, () => {
-                const documentDeclaration = service.getDocumentDeclaration(type);
+                const terms = service.getTerms(type);
 
-                documentDeclaration.pages.forEach(page => {
-                  let content;
+                terms.sourceDocuments.forEach(sourceDocument => {
                   let filteredContent;
-                  let mimeType;
 
-                  context(page.location, () => {
+                  context(sourceDocument.location, () => {
                     before(async function () {
-                      if (!documentDeclaration) {
+                      if (!terms) {
                         console.log('      (Tests skipped as declaration has been archived)');
                         this.skip();
                       }
                     });
 
                     it('fetchable URL', async () => {
-                      const { location, executeClientScripts } = page;
-                      const document = await fetch({
+                      const { location, executeClientScripts } = sourceDocument;
+
+                      ({ content: sourceDocument.content, mimeType: sourceDocument.mimeType } = await fetch({
                         url: location,
                         executeClientScripts,
-                        cssSelectors: page.cssSelectors,
+                        cssSelectors: sourceDocument.cssSelectors,
                         config: config.get('fetcher'),
-                      });
-
-                      content = document.content;
-                      mimeType = document.mimeType;
+                      }));
                     });
 
-                    it('selector matches an element in the web page', async function checkSelector() {
-                      if (!content) {
+                    it('selector matches an element in the source document', async function checkSelector() {
+                      if (!sourceDocument.content) {
                         console.log('          [Tests skipped as URL is not fetchable]');
                         this.skip();
                       }
 
-                      filteredContent = await filter({ content, pageDeclaration: page, mimeType });
+                      filteredContent = await extract(sourceDocument);
 
                       expect(filteredContent).to.not.be.empty;
                     });
 
                     it(`filtered content has at least ${MIN_DOC_LENGTH} characters`, async function checkContentLength() {
-                      if (!content) {
+                      if (!sourceDocument.content) {
                         console.log('          [Tests skipped as URL is not fetchable]');
                         this.skip();
                       }
@@ -146,7 +142,7 @@ export default async options => {
                     it('content is consistent when fetched and filtered twice in a row', async function checkContentConsistency() {
                       this.slow(SLOW_DOCUMENT_THRESHOLD * 2);
 
-                      if (!content) {
+                      if (!sourceDocument.content) {
                         console.log('          [Tests skipped as URL is not fetchable]');
                         this.skip();
                       }
@@ -156,13 +152,13 @@ export default async options => {
                         this.skip();
                       }
 
-                      const document = await fetch({
-                        url: page.location,
-                        executeClientScripts: page.executeClientScripts,
-                        cssSelectors: page.cssSelectors,
+                      ({ content: sourceDocument.content, mimeType: sourceDocument.mimeType } = await fetch({
+                        url: sourceDocument.location,
+                        executeClientScripts: sourceDocument.executeClientScripts,
+                        cssSelectors: sourceDocument.cssSelectors,
                         config: config.get('fetcher'),
-                      });
-                      const secondFilteredContent = await filter({ content: document.content, pageDeclaration: page, mimeType: document.mimeType });
+                      }));
+                      const secondFilteredContent = await extract(sourceDocument);
 
                       expect(secondFilteredContent).to.equal(filteredContent);
                     });

package/scripts/declarations/validate/service.history.schema.js

@@ -1,10 +1,10 @@
-import { DOCUMENT_TYPES } from '../../../src/archivist/services/index.js';
+import TERMS_TYPES from '@opentermsarchive/terms-types';
 
 import definitions from './definitions.js';
 
-const AVAILABLE_TYPES_NAME = Object.keys(DOCUMENT_TYPES);
+const AVAILABLE_TYPES_NAME = Object.keys(TERMS_TYPES);
 
-const documentsProperties = () => {
+const termsProperties = () => {
   const result = {};
 
   AVAILABLE_TYPES_NAME.forEach(type => {
@@ -12,8 +12,8 @@ const documentsProperties = () => {
       type: 'array',
       items: {
         oneOf: [
-          { $ref: '#/definitions/singlePageDocumentHistory' },
-          { $ref: '#/definitions/multiPageDocumentHistory' },
+          { $ref: '#/definitions/singleSourceDocumentTermsHistory' },
+          { $ref: '#/definitions/multipleSourceDocumentsTermsHistory' },
           { $ref: '#/definitions/pdfDocumentHistory' },
         ],
       },
@@ -27,7 +27,7 @@ const schema = {
   type: 'object',
   additionalProperties: false,
   title: 'Service declaration history',
-  properties: documentsProperties(),
+  properties: termsProperties(),
   propertyNames: { enum: AVAILABLE_TYPES_NAME },
   definitions: {
     ...definitions,
@@ -40,7 +40,7 @@ const schema = {
         validUntil: { $ref: '#/definitions/validUntil' },
       },
     },
-    singlePageDocumentHistory: {
+    singleSourceDocumentTermsHistory: {
       type: 'object',
       additionalProperties: false,
       required: [ 'fetch', 'select', 'validUntil' ],
@@ -48,12 +48,12 @@ const schema = {
         fetch: { $ref: '#/definitions/location' },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
         validUntil: { $ref: '#/definitions/validUntil' },
       },
     },
-    multiPageDocumentHistory: {
+    multipleSourceDocumentsTermsHistory: {
       type: 'object',
       additionalProperties: false,
       required: ['combine'],
@@ -68,14 +68,14 @@ const schema = {
               fetch: { $ref: '#/definitions/location' },
               select: { $ref: '#/definitions/contentSelectors' },
               filter: { $ref: '#/definitions/filters' },
-              remove: { $ref: '#/definitions/noiseSelectors' },
+              remove: { $ref: '#/definitions/insignificantContentSelectors' },
               executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
             },
           },
         },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
         validUntil: { $ref: '#/definitions/validUntil' },
       },

package/scripts/declarations/validate/service.schema.js

@@ -1,17 +1,17 @@
-import { DOCUMENT_TYPES } from '../../../src/archivist/services/index.js';
+import TERMS_TYPES from '@opentermsarchive/terms-types';
 
 import definitions from './definitions.js';
 
-const AVAILABLE_TYPES_NAME = Object.keys(DOCUMENT_TYPES);
+const AVAILABLE_TYPES_NAME = Object.keys(TERMS_TYPES);
 
-const documentsProperties = () => {
+const termsProperties = () => {
   const result = {};
 
   AVAILABLE_TYPES_NAME.forEach(type => {
     result[type] = {
       oneOf: [
-        { $ref: '#/definitions/singlePageDocument' },
-        { $ref: '#/definitions/multiPageDocument' },
+        { $ref: '#/definitions/singleSourceDocumentTerms' },
+        { $ref: '#/definitions/multipleSourceDocumentsTerms' },
         { $ref: '#/definitions/pdfDocument' },
       ],
     };
@@ -33,7 +33,7 @@ const schema = {
     },
     documents: {
       type: 'object',
-      properties: documentsProperties(),
+      properties: termsProperties(),
       propertyNames: { enum: AVAILABLE_TYPES_NAME },
     },
     importedFrom: {
@@ -52,7 +52,7 @@ const schema = {
       required: ['fetch'],
       properties: { fetch: { $ref: '#/definitions/pdfLocation' } },
     },
-    page: {
+    sourceDocument: {
       type: 'object',
       additionalProperties: false,
       required: ['fetch'],
@@ -60,28 +60,28 @@ const schema = {
         fetch: { $ref: '#/definitions/location' },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
       },
     },
-    singlePageDocument: {
+    singleSourceDocumentTerms: {
       allOf: [
-        { $ref: '#/definitions/page' },
+        { $ref: '#/definitions/sourceDocument' },
         { required: [ 'fetch', 'select' ] },
       ],
     },
-    multiPageDocument: {
+    multipleSourceDocumentsTerms: {
       type: 'object',
       additionalProperties: false,
       required: ['combine'],
       properties: {
         combine: {
           type: 'array',
-          items: { $ref: '#/definitions/page' },
+          items: { $ref: '#/definitions/sourceDocument' },
         },
         select: { $ref: '#/definitions/contentSelectors' },
         filter: { $ref: '#/definitions/filters' },
-        remove: { $ref: '#/definitions/noiseSelectors' },
+        remove: { $ref: '#/definitions/insignificantContentSelectors' },
         executeClientScripts: { $ref: '#/definitions/executeClientScripts' },
       },
     },

package/scripts/history/migrate-services.js

@@ -145,10 +145,10 @@ async function rewriteSnapshots(repository, records, idsMapping, logger) {
     idsMapping[record.id] = recordId; // Saves the mapping between the old ID and the new one.
 
     if (recordId) {
-      logger.info({ message: `Migrated snapshot with new ID: ${recordId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: `Migrated snapshot with new ID: ${recordId}`, serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.migrated++;
     } else {
-      logger.info({ message: 'Skipped snapshot', serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: 'Skipped snapshot', serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.skipped++;
     }
   }
@@ -169,10 +169,10 @@ async function rewriteVersions(repository, records, idsMapping, logger) {
     const { id: recordId } = await repository.save(record); // eslint-disable-line no-await-in-loop
 
     if (recordId) {
-      logger.info({ message: `Migrated version with new ID: ${recordId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: `Migrated version with new ID: ${recordId}`, serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.migrated++;
     } else {
-      logger.info({ message: 'Skipped version', serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      logger.info({ message: 'Skipped version', serviceId: record.serviceId, type: record.termsType, id: record.id, current: i++, total: records.length });
       counters.skipped++;
     }
   }

package/scripts/history/update-to-full-hash.js

@@ -47,9 +47,9 @@ const ROOT_PATH = path.resolve(__dirname, '../../');
     const { id: recordId } = await versionsTargetRepository.save(record);
 
     if (!recordId) {
-      logger.warn({ message: 'Record skipped', serviceId: record.serviceId, type: record.documentType, id: record.id, current, total });
+      logger.warn({ message: 'Record skipped', serviceId: record.serviceId, type: record.termsType, id: record.id, current, total });
     } else {
-      logger.info({ message: `Update short sha ${record.snapshotId} to ${fullSnapshotId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current, total });
+      logger.info({ message: `Update short sha ${record.snapshotId} to ${fullSnapshotId}`, serviceId: record.serviceId, type: record.termsType, id: record.id, current, total });
     }
 
     current++;

package/scripts/import/index.js

@@ -18,7 +18,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const ROOT_PATH = path.resolve(__dirname, '../../');
 const MAX_PARALLEL = 10;
 const MAX_RETRY = 5;
-const PDF_MIME_TYPE = 'application/pdf';
+const PDF_MIME_TYPE = mime.getType('pdf');
 const COUNTERS = {
   imported: 0,
   skippedNoChanges: 0,
@@ -87,10 +87,10 @@ function queueErrorHandler(error, { commit }) {
 
   const serviceId = path.dirname(relativeFilePath);
   const extension = path.extname(relativeFilePath);
-  const documentType = path.basename(relativeFilePath, extension);
+  const termsType = path.basename(relativeFilePath, extension);
 
   commitsNotImported.push(commit.hash);
-  logger.error({ message: `${error.stack}\nCommit details: ${JSON.stringify(commit, null, 2)}`, serviceId, type: documentType, sha: commit.hash });
+  logger.error({ message: `${error.stack}\nCommit details: ${JSON.stringify(commit, null, 2)}`, serviceId, type: termsType, sha: commit.hash });
   COUNTERS.errors++;
 }
 
@@ -117,9 +117,9 @@ function queueDrainHandler(totalToTreat) {
   };
 }
 
-async function getCommitContent({ sha, serviceId, documentType, extension }) {
+async function getCommitContent({ sha, serviceId, termsType, extension }) {
   const start = performance.now();
-  const url = `https://raw.githubusercontent.com/${config.get('import.githubRepository')}/${sha}/${encodeURI(serviceId)}/${encodeURI(documentType)}.${extension}`;
+  const url = `https://raw.githubusercontent.com/${config.get('import.githubRepository')}/${sha}/${encodeURI(serviceId)}/${encodeURI(termsType)}.${extension}`;
   const response = await nodeFetch(url);
   const end = performance.now();
 
@@ -141,7 +141,7 @@ async function getCommitContent({ sha, serviceId, documentType, extension }) {
     throw new TooManyRequestsError(`Cannot get commit content on Github ${url}. 429: Too Many Requests`);
   }
 
-  logger.info({ message: `Fetched in ${Number(end - start).toFixed(2)} ms`, serviceId, type: documentType, sha });
+  logger.info({ message: `Fetched in ${Number(end - start).toFixed(2)} ms`, serviceId, type: termsType, sha });
 
   return content;
 }
@@ -151,12 +151,12 @@ async function handleCommit(commit, index, total) {
 
   let serviceId = path.dirname(relativeFilePath);
   const extension = path.extname(relativeFilePath);
-  let documentType = path.basename(relativeFilePath, extension);
+  let termsType = path.basename(relativeFilePath, extension);
 
   logger.info({
     message: 'Start to handle commit',
     serviceId,
-    type: documentType,
+    type: termsType,
     sha: commit.hash,
     current: index + 1,
     total,
@@ -168,7 +168,7 @@ async function handleCommit(commit, index, total) {
     logger.info({
       message: 'Skipped commit as an entry already exists for this commit',
       serviceId,
-      type: documentType,
+      type: termsType,
       sha: commit.hash,
     });
     COUNTERS.skippedNoChanges++;
@@ -176,9 +176,9 @@ async function handleCommit(commit, index, total) {
     return;
   }
 
-  let content = await getCommitContent({ sha: commit.hash, serviceId, documentType, extension: extension.replace('.', '') });
+  let content = await getCommitContent({ sha: commit.hash, serviceId, termsType, extension: extension.replace('.', '') });
 
-  ({ serviceId, documentType } = renamer.applyRules(serviceId, documentType));
+  ({ serviceId, termsType } = renamer.applyRules(serviceId, termsType));
 
   const mimeType = mime.getType(extension);
 
@@ -198,7 +198,7 @@ async function handleCommit(commit, index, total) {
 
     await snapshotsCollection.insertOne({
       serviceId,
-      documentType,
+      termsType,
       content,
       mimeType,
       fetchDate: commit.date,
@@ -207,10 +207,10 @@ async function handleCommit(commit, index, total) {
     });
     const end = performance.now();
 
-    logger.info({ message: `Recorded in ${Number(end - start).toFixed(2)} ms`, serviceId, type: documentType });
+    logger.info({ message: `Recorded in ${Number(end - start).toFixed(2)} ms`, serviceId, type: termsType });
     COUNTERS.imported++;
   } catch (error) {
-    logger.error({ message: `Not saved: ${commit.date} ${error.stack}`, serviceId, type: documentType });
+    logger.error({ message: `Not saved: ${commit.date} ${error.stack}`, serviceId, type: termsType });
     commitsNotImported.push(commit.hash);
     COUNTERS.errors++;
   }

package/scripts/rewrite/config/rewrite-snapshots.json

@@ -5,7 +5,7 @@
         "git": {
           "path": "./data/versions",
           "publish": false,
-          "prefixMessageToSnapshotId": "This version was recorded after filtering snapshot https://github.com/ambanum/OpenTermsArchive-snapshots/commit/",
+          "prefixMessageToSnapshotId": "This version was recorded after filtering snapshot https://github.com/OpenTermsArchive/contrib-snapshots/commit/",
           "author": {
             "name": "Open Terms Archive Bot",
             "email": "bot@opentermsarchive.org"

package/scripts/rewrite/config/rewrite-versions.json

@@ -5,7 +5,7 @@
         "git": {
           "path": "./data/versions-rewritten",
           "publish": false,
-          "prefixMessageToSnapshotId": "This version was recorded after filtering snapshot https://github.com/ambanum/OpenTermsArchive-snapshots/commit/",
+          "prefixMessageToSnapshotId": "This version was recorded after filtering snapshot https://github.com/OpenTermsArchive/contrib-snapshots/commit/",
           "author": {
             "name": "Open Terms Archive Bot",
             "email": "bot@opentermsarchive.org"

package/scripts/rewrite/rewrite-snapshots.js

@@ -76,13 +76,13 @@ let recorder;
     const { content, mimeType } = await loadFile(SNAPSHOTS_SOURCE_PATH, relativeFilePath);
 
     let serviceId = path.dirname(relativeFilePath);
-    let documentType = path.basename(relativeFilePath, path.extname(relativeFilePath));
+    let termsType = path.basename(relativeFilePath, path.extname(relativeFilePath));
 
-    ({ serviceId, documentType } = renamer.applyRules(serviceId, documentType));
+    ({ serviceId, termsType } = renamer.applyRules(serviceId, termsType));
 
     const { id: snapshotId } = await recorder.recordSnapshot({
       serviceId,
-      documentType,
+      termsType,
       content,
       mimeType,
       fetchDate: commit.date,