@opentermsarchive/engine 0.15.0

package/scripts/rewrite/rewrite-versions.js

@@ -0,0 +1,160 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import config from 'config';
+
+import { InaccessibleContentError } from '../../src/archivist/errors.js';
+import filter from '../../src/archivist/filter/index.js';
+import Recorder from '../../src/archivist/recorder/index.js';
+import Git from '../../src/archivist/recorder/repositories/git/git.js';
+import GitRepository from '../../src/archivist/recorder/repositories/git/index.js';
+import * as services from '../../src/archivist/services/index.js';
+import * as renamer from '../utils/renamer/index.js';
+
+import * as initializer from './initializer/index.js';
+import { loadFile } from './utils.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const ROOT_PATH = path.resolve(__dirname, '../../');
+const MARKDOWN_MIME_TYPE = 'text/markdown';
+
+export const SNAPSHOTS_SOURCE_PATH = path.resolve(
+  ROOT_PATH,
+  config.get('rewrite.snapshotsSourcePath'),
+);
+export const VERSIONS_TARGET_PATH = path.resolve(ROOT_PATH, config.get('recorder.versions.storage.git.path'));
+
+const initialize = process.argv.includes('--init');
+
+const COUNTERS = {
+  rewritten: 0,
+  skippedNoChanges: 0,
+  skippedInaccessibleContent: 0,
+  skippedUnknownError: 0,
+};
+
+let recorder;
+
+(async () => {
+  console.time('Total time');
+  console.log('Start rewritting history.');
+
+  await renamer.loadRules();
+  const servicesDeclarations = await services.loadWithHistory();
+  const sourceRepo = new Git({ path: SNAPSHOTS_SOURCE_PATH, author: config.get('recorder.snapshots.storage.git.author') });
+
+  await sourceRepo.initialize();
+
+  console.log('Waiting for git log… (this can take a while)');
+  const commits = (await sourceRepo.log(['--stat=4096'])).sort((a, b) => new Date(a.date) - new Date(b.date));
+
+  console.log(`Source repo contains ${commits.length} commits.\n`);
+
+  if (initialize) {
+    const targetRepo = await initializer.initTargetRepo(VERSIONS_TARGET_PATH);
+    const [readmeCommit] = commits;
+
+    await initializer.initReadmeAndLicense(targetRepo, VERSIONS_TARGET_PATH, readmeCommit.date);
+  }
+
+  recorder = new Recorder({
+    versionsRepository: new GitRepository({
+      ...config.get('recorder.versions.storage.git'),
+      path: VERSIONS_TARGET_PATH,
+    }),
+    snapshotsRepository: new GitRepository({
+      ...config.get('recorder.snapshots.storage.git'),
+      path: SNAPSHOTS_SOURCE_PATH,
+    }),
+  });
+
+  await recorder.initialize();
+
+  const filteredCommits = commits.filter(({ message }) =>
+    message.match(/^(Start tracking|Update)/));
+
+  /* eslint-disable no-await-in-loop */
+  /* eslint-disable no-continue */
+  for (const commit of filteredCommits) {
+    console.log(Date.now(), commit.hash, commit.date, commit.message);
+
+    await sourceRepo.checkout(commit.hash);
+
+    const [{ file: relativeFilePath }] = commit.diff.files;
+
+    const { content, mimeType } = await loadFile(SNAPSHOTS_SOURCE_PATH, relativeFilePath);
+
+    let serviceId = path.dirname(relativeFilePath);
+    let documentType = path.basename(relativeFilePath, path.extname(relativeFilePath));
+
+    ({ serviceId, documentType } = renamer.applyRules(serviceId, documentType));
+
+    if (!servicesDeclarations[serviceId]) {
+      console.log(`⌙ Skip unknown service "${serviceId}"`);
+      continue;
+    }
+
+    const documentDeclaration = servicesDeclarations[serviceId].getDocumentDeclaration(
+      documentType,
+      commit.date,
+    );
+
+    if (!documentDeclaration) {
+      console.log(`⌙ Skip unknown document type "${documentType}" for service "${serviceId}"`);
+      continue;
+    }
+
+    if (documentDeclaration.validUntil) {
+      console.log(`⌙ Use declaration valid until ${documentDeclaration.validUntil}`);
+    }
+
+    try {
+      const document = await filter({
+        content,
+        mimeType,
+        documentDeclaration,
+      });
+
+      const { id: versionId } = await recorder.recordVersion({
+        serviceId,
+        documentType,
+        content: document,
+        mimeType: MARKDOWN_MIME_TYPE, // The result of the `filter` function is always in markdown format
+        fetchDate: commit.date,
+        snapshotId: commit.hash,
+      });
+
+      if (versionId) {
+        COUNTERS.rewritten++;
+      } else {
+        COUNTERS.skippedNoChanges++;
+      }
+    } catch (error) {
+      if (error instanceof InaccessibleContentError) {
+        console.log('⌙ Skip inacessible content');
+        COUNTERS.skippedInaccessibleContent++;
+      } else {
+        console.log('⌙ Unknown error:', error);
+        COUNTERS.skippedUnknownError++;
+      }
+    }
+  }
+
+  const totalTreatedCommits = Object.values(COUNTERS).reduce((acc, value) => acc + value, 0);
+
+  console.log(`\nCommits treated: ${totalTreatedCommits} on ${filteredCommits.length}`);
+  console.log(`⌙ Commits rewritten: ${COUNTERS.rewritten}`);
+  console.log(`⌙ Skipped not changed commits: ${COUNTERS.skippedNoChanges}`);
+  console.log(`⌙ Skipped inacessible content: ${COUNTERS.skippedInaccessibleContent}`);
+  console.log(`⌙ Skipped unknown error: ${COUNTERS.skippedUnknownError}`);
+  console.timeEnd('Total time');
+
+  if (totalTreatedCommits != filteredCommits.length) {
+    console.error('\n⚠ WARNING: Total treated commits does not match the total number of commits to be treated! ⚠');
+  }
+
+  if (COUNTERS.skippedUnknownError) {
+    console.error('\n⚠ WARNING: Some unknown errors occured, check log! ⚠');
+  }
+})();

package/scripts/rewrite/utils.js

@@ -0,0 +1,33 @@
+import fsApi from 'fs';
+
+import mime from 'mime';
+
+const fs = fsApi.promises;
+
+export async function fileExists(filePath) {
+  try {
+    await fs.access(filePath);
+
+    return true;
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return false;
+    }
+  }
+}
+
+export async function loadFile(repoPath, relativeFilePath) {
+  const absoluteFilePath = `${repoPath}/${relativeFilePath}`;
+
+  const mimeType = mime.getType(absoluteFilePath);
+  const readFileOptions = {};
+
+  if (mimeType.startsWith('text/')) {
+    readFileOptions.encoding = 'utf8';
+  }
+
+  return {
+    content: await fs.readFile(absoluteFilePath, readFileOptions),
+    mimeType,
+  };
+}

package/scripts/utils/renamer/README.md

@@ -0,0 +1,49 @@
+# Renamer
+
+This module is used to apply renaming rules to service IDs and document types.
+
+## Usage
+
+You can use it in your other scripts like this:
+
+```
+await renamer.loadRules();
+const { serviceId: renamedServiceId, documentType: renamedDocumentType } = renamer.applyRules(serviceId, documentType);
+```
+
+## Adding renaming rules
+
+### Service
+
+To rename a service, add a rule in `./rules/services.json`, for example, to rename "GoogleAds" to "Google Ads", add the following line in the file:
+
+```json
+{
+  …
+  "GoogleAds": "Google Ads"
+}
+```
+
+### Document type
+
+To rename a document type, add a rule in `./rules/documentTypes.json`, for example, to rename "Program Policies" to "Acceptable Use Policy", add the following line in the file:
+
+```json
+{
+  …
+  "Program Policies": "Acceptable Use Policy"
+}
+```
+
+### Document type for a specific service
+
+To rename a document type only for a specific service, add a rule in `./rules/servicesDocumentTypes.json`, for example, to rename "Program Policies" to "Acceptable Use Policy" only for Skype, add the following line in the file:
+
+```json
+{
+  …
+  "Skype": {
+    "Program Policies": "Acceptable Use Policy"
+  }
+}
+```

package/scripts/utils/renamer/index.js

@@ -0,0 +1,45 @@
+import fsApi from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const fs = fsApi.promises;
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+let renamingRules;
+
+export async function loadRules() {
+  renamingRules = {
+    serviceNames: JSON.parse(await fs.readFile(path.resolve(__dirname, './rules/serviceNames.json'))),
+    documentTypes: JSON.parse(await fs.readFile(path.resolve(__dirname, './rules/documentTypes.json'))),
+    documentTypesByService: JSON.parse(await fs.readFile(path.resolve(__dirname, './rules/documentTypesByService.json'))),
+  };
+}
+
+export function applyRules(serviceId, documentType) {
+  const renamedServiceId = renamingRules.serviceNames[serviceId];
+
+  if (renamedServiceId) {
+    console.log(`⌙ Rename service "${serviceId}" to "${renamedServiceId}"`);
+    serviceId = renamedServiceId;
+  }
+
+  const renamedDocumentType = renamingRules.documentTypes[documentType];
+
+  if (renamedDocumentType) {
+    console.log(`⌙ Rename document type "${documentType}" to "${renamedDocumentType}" of "${serviceId}" service`);
+    documentType = renamedDocumentType;
+  }
+
+  const renamedServiceDocumentType = renamingRules.documentTypesByService[serviceId]
+    && renamingRules.documentTypesByService[serviceId][documentType];
+
+  if (renamedServiceDocumentType) {
+    console.log(`⌙ Specific rename document type "${documentType}" to "${renamedServiceDocumentType}" of "${serviceId}" service`);
+    documentType = renamedServiceDocumentType;
+  }
+
+  return {
+    serviceId,
+    documentType,
+  };
+}

package/scripts/utils/renamer/rules/documentTypes.json

@@ -0,0 +1,25 @@
+{
+  "acceptable_use_policy": "Acceptable Use Policy",
+  "brand_guidelines": "Brand Guidelines",
+  "commercial_terms": "Commercial Terms",
+  "community_guidelines": "Community Guidelines",
+  "controller_controller_data_protection_terms": "Data Controller Agreement",
+  "Cookies Policy": "Trackers Policy",
+  "cookies_policy": "Trackers Policy",
+  "copyright_policy": "Copyright Claims Policy",
+  "data_processing_terms": "Data Processor Agreement",
+  "developer_agreement": "Developer Terms",
+  "developer_policy": "Developer Terms",
+  "in_app_purchases_policy": "In-App Purchases Policy",
+  "law_enforcement_guidelines": "Law Enforcement Guidelines",
+  "privacy_policy": "Privacy Policy",
+  "Program Policies": "Acceptable Use Policy",
+  "review_guidelines": "Review Guidelines",
+  "software_license_agreement": "Software License Agreement",
+  "Terms Of Service": "Terms of Service",
+  "terms_of_service_parent_company": "Parent Organization Terms",
+  "terms_of_service": "Terms of Service",
+  "tos_parent": "Parent Organization Terms",
+  "user_consent_policy": "User Consent Policy",
+  "WEBSITE TERMS OF USE": "Website Terms of Use"
+}

package/scripts/utils/renamer/rules/documentTypesByService.json

@@ -0,0 +1,170 @@
+{
+  "Skype": {
+    "undefined": "Parent Organization Terms"
+  },
+  "Aegean Airlines": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Aigle": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Air Corsica": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Air Europa": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "BackMarket": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Benetton": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Bershka": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Bizzbee": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "BlaBlaCar": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Boulanger": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Brice": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "C-A": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Camaieu": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Caroll": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Cdiscount": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Comptoir des Cotonniers": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Cop-Copine": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Damart": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Darty": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Decathlon": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Devred": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "E.Leclerc": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Esprit": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Etam": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Eurolines": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "FlixBus": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "G7": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Jennyfer": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Jules": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Kaporal": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Kappa": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Kookai": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "La Redoute": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Lacoste": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Lafuma": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Le Coq Sportif": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "LeCab": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Levis": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Morgan": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Naf Naf": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "New Balance": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Nike": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Printemps": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Rue du Commerce": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "SHEIN": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "SNCF Connect": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Salto": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Showroomprive.com": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Spartoo": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Tunisair": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Vans": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Veepee": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Verbaudet": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Vinted": {
+    "Commercial Terms": "General Conditions of Sale"
+  },
+  "Zalando": {
+    "Commercial Terms": "General Conditions of Sale"
+  }
+}

package/scripts/utils/renamer/rules/serviceNames.json

@@ -0,0 +1,92 @@
+{
+  "123 Greetings": "123Greetings",
+  "Academia": "Academia.edu",
+  "Alibaba": "Alibaba.com",
+  "Amazon": "Amazon.com",
+  "AmazonAlexa": "Alexa",
+  "AmazonAmazonAppStoreAndroid": "Amazon Appstore",
+  "AmazonCoins": "Amazon Coins",
+  "AmazonDevice": "Amazon Device",
+  "AmazonDriveAndPrimePhotos": "Amazon Photos",
+  "AmazonKindleStore": "Amazon Kindle Store",
+  "AmazonMaps": "Amazon Maps",
+  "AmazonMonthlyPayments": "Amazon Monthly Payments",
+  "AmazonMusic": "Amazon Music",
+  "AmazonPrime": "Amazon Prime",
+  "AmazonSilk": "Amazon Silk",
+  "AppleAppStore": "App Store",
+  "AppleGameCenter": "Apple Game Center",
+  "AppleICloud": "iCloud",
+  "AskFM": "ASKfm",
+  "Asus": "ASUS",
+  "Bbc": "BBC",
+  "Blizzard": "Blizzard Entertainment",
+  "Cnn": "CNN International",
+  "Commentcamarche": "CommentCaMarche",
+  "Cvs": "CVS Pharmacy",
+  "Dailymail": "Daily Mail",
+  "deviantART": "DeviantArt",
+  "Discordapp": "Discord",
+  "Duckduckgo": "DuckDuckGo",
+  "Ebay": "eBay",
+  "FacebookAds": "Facebook Ads",
+  "FacebookPayments": "Facebook Payments",
+  "Fedex": "FedEx",
+  "Foxnews": "Fox News",
+  "Ft": "Financial Times",
+  "Github": "GitHub",
+  "Gitlab": "GitLab",
+  "GoogleAdMob": "AdMob",
+  "GoogleAds": "Google Ads",
+  "GoogleAdSense": "AdSense",
+  "GoogleAnalytics": "Google Analytics",
+  "GooglePlayStore": "Google Play",
+  "Hsbc": "HSBC",
+  "Ign": "IGN Entertainment",
+  "Imdb": "IMDb",
+  "LastFm": "Last.fm",
+  "LinguaLeo": "Lingualeo",
+  "Linkedin": "LinkedIn",
+  "Microsoftstore": "Microsoft Store",
+  "Msn": "MSN",
+  "Nationalgeographic": "National Geographic",
+  "Nytimes": "The New York Times",
+  "Okcupid": "OkCupid",
+  "Oreilly": "O'Reilly",
+  "Reuters": "thomsonreuters.com",
+  "Shockwave": "Shockwave.com",
+  "Sonic": "Sonic.net",
+  "Sony": "sony.com",
+  "StackOverflow": "Stack Overflow",
+  "Surveymonkey": "SurveyMonkey",
+  "Theguardian": "The Guardian",
+  "Theregister": "The Register",
+  "Time": "TIME",
+  "Timeanddate": "timeanddate.com",
+  "Toyota": "Toyota Connected Services",
+  "Tp-link": "TP-Link",
+  "Ubuntu": "ubuntu.com",
+  "Ucla": "ucla.edu",
+  "Ui": "Ubiquiti",
+  "Ulster": "ulster.ac.uk",
+  "Urbandictionary": "UrbanDictionary.com",
+  "Usa": "USA.gov",
+  "Veracitypayments": "Vanco Payment Solutions",
+  "Veranda": "Hearst.com",
+  "W3schools": "W3Schools",
+  "Waterhavens": "WaterHavens",
+  "Webcrawler": "WebCrawler",
+  "Webpronews": "WebProNews",
+  "WeChatOpenPlatform": "WeChat Open Platform",
+  "WeHeartIt": "We Heart It",
+  "Wikimediafoundation": "Wikimedia Foundation",
+  "Wineverygame": "WinEveryGame",
+  "Wolframalpha": "WolframAlpha",
+  "Womansday": "Womans Day",
+  "Wordpress": "WordPress.com",
+  "Worldmarket": "World Market",
+  "XfinityResidentialSubscription": "Xfinity Residential Services",
+  "XfinityWebServices": "Xfinity Web Services",
+  "Youtube": "YouTube",
+  "Verbaudet": "Vertbaudet"
+}

package/src/archivist/errors.js

@@ -0,0 +1,9 @@
+export class InaccessibleContentError extends Error {
+  constructor(message) {
+    if (Array.isArray(message)) {
+      message = `\n - ${message.join('\n - ')}`;
+    }
+    super(`The document cannot be accessed or its content can not be selected:${message}`);
+    this.name = 'InaccessibleContentError';
+  }
+}

package/src/archivist/fetcher/errors.js

@@ -0,0 +1,6 @@
+export class FetchDocumentError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'FetchDocumentError';
+  }
+}

package/src/archivist/fetcher/exports.js

@@ -0,0 +1,18 @@
+import '../../../bin/.env.js'; // Workaround to ensure `SUPPRESS_NO_CONFIG_WARNING` is set before config is imported
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import config from 'config';
+
+import fetcher from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const defaultConfigs = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../../../config/default.json')));
+
+config.util.setModuleDefaults('fetcher', defaultConfigs.fetcher);
+
+export { launchHeadlessBrowser, stopHeadlessBrowser } from './index.js';
+
+export default fetcher;