@opentermsarchive/engine 0.15.0

package/scripts/history/logger/index.js

@@ -0,0 +1,39 @@
+import winston from 'winston';
+
+import logger from '../../../src/logger/index.js';
+
+const { combine, timestamp, printf, colorize } = winston.format;
+
+export const format = combine(
+  colorize(),
+  timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  printf(({ level, message, timestamp, serviceId, type, id, current, total }) => {
+    let prefix = ''.padEnd(8);
+
+    if (current && total) {
+      prefix = `${Number(((current) / total) * 100).toFixed(2)}%`.padEnd(8);
+    }
+
+    if (serviceId) {
+      prefix += `${serviceId}`.padEnd(30);
+    }
+
+    if (type) {
+      if (type.length > 50) {
+        type = `${type.substring(0, 48)}…`;
+      }
+
+      prefix += `${type}`.padEnd(50);
+    }
+
+    if (id) {
+      prefix += `${id}`.padEnd(42);
+    }
+
+    return `${timestamp} ${level.padEnd(15)} ${prefix}${message}`;
+  }),
+);
+
+logger.format = format;
+
+export default logger;

package/scripts/history/migrate-services.js

@@ -0,0 +1,212 @@
+import fsApi from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import config from 'config';
+import winston from 'winston';
+
+import GitRepository from '../../src/archivist/recorder/repositories/git/index.js';
+
+import { format } from './logger/index.js';
+import { importReadmeInGit } from './utils/index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT_PATH = path.resolve(__dirname, '../../');
+const fs = fsApi.promises;
+
+const CONFIG = {
+  servicesToMigrate: [ 'ASICS', 'Amazon', 'Orange Money France' ],
+  from: {
+    snapshots: 'france-snapshots',
+    versions: 'france-versions-hash-updated',
+    prefixMessageToSnapshotId: 'This version was recorded after filtering snapshot https://github.com/OpenTermsArchive/france-snapshots/commit/',
+  },
+  to: {
+    snapshots: 'france-elections-snapshots',
+    versions: 'france-elections-versions-hash-updated',
+    prefixMessageToSnapshotId: 'This version was recorded after filtering snapshot https://github.com/OpenTermsArchive/france-elections-snapshots/commit/',
+  },
+};
+
+const counters = {
+  migrated: 0,
+  skipped: 0,
+};
+
+(async function main() {
+  console.time('Total time');
+
+  const migration = {
+    services: CONFIG.servicesToMigrate,
+    from: {
+      snapshots: {
+        source: new GitRepository({
+          ...config.get('recorder.snapshots.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.snapshots}`),
+        }),
+        destination: new GitRepository({
+          ...config.get('recorder.snapshots.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.snapshots}-migrated`),
+        }),
+        logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.from.snapshots}.log` }), new winston.transports.Console() ], format }),
+      },
+      versions: {
+        source: new GitRepository({
+          ...config.get('recorder.versions.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.versions}`),
+        }),
+        destination: new GitRepository({
+          ...config.get('recorder.versions.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.from.versions}-migrated`),
+          prefixMessageToSnapshotId: CONFIG.from.prefixMessageToSnapshotId,
+        }),
+        logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.from.versions}.log` }), new winston.transports.Console() ], format }),
+      },
+    },
+    to: {
+      snapshots: {
+        source: new GitRepository({
+          ...config.get('recorder.snapshots.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.snapshots}`),
+        }),
+        destination: new GitRepository({
+          ...config.get('recorder.snapshots.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.snapshots}-migrated`),
+        }),
+        logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.to.snapshots}.log` }), new winston.transports.Console() ], format }),
+      },
+      versions: {
+        source: new GitRepository({
+          ...config.get('recorder.versions.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.versions}`),
+        }),
+        destination: new GitRepository({
+          ...config.get('recorder.versions.storage.git'),
+          path: path.resolve(ROOT_PATH, `./data/${CONFIG.to.versions}-migrated`),
+          prefixMessageToSnapshotId: CONFIG.to.prefixMessageToSnapshotId,
+        }),
+        logger: winston.createLogger({ transports: [ new (winston.transports.File)({ filename: `${__dirname}/logs/${CONFIG.to.versions}.log` }), new winston.transports.Console() ], format }),
+      },
+    },
+  };
+
+  await initialize(migration);
+
+  const fromSnapshotsRecords = await migration.from.snapshots.source.findAll();
+  const toSnapshotsRecords = await migration.to.snapshots.source.findAll();
+  const snapshotsToMigrate = fromSnapshotsRecords.filter(({ serviceId }) => migration.services.includes(serviceId));
+  const fromSnapshotsRecordsToRewrite = fromSnapshotsRecords.filter(({ serviceId }) => !migration.services.includes(serviceId));
+  const toSnapshotsRecordsMigrated = [ ...toSnapshotsRecords, ...snapshotsToMigrate ].sort((recordA, recordB) => new Date(recordA.fetchDate) - new Date(recordB.fetchDate));
+
+  const fromVersionsRecords = await migration.from.versions.source.findAll();
+  const toVersionsRecords = await migration.to.versions.source.findAll();
+  const versionsToMigrate = fromVersionsRecords.filter(({ serviceId }) => migration.services.includes(serviceId));
+  const fromVersionsRecordsToRewrite = fromVersionsRecords.filter(({ serviceId }) => !migration.services.includes(serviceId));
+  const toVersionsRecordsMigrated = [ ...toVersionsRecords, ...versionsToMigrate ].sort((recordA, recordB) => new Date(recordA.fetchDate) - new Date(recordB.fetchDate));
+
+  console.log('Number of snapshots in the source', fromSnapshotsRecords.length);
+  console.log('Number of snapshots in the target', toSnapshotsRecords.length);
+  console.log('Number of snapshots to migrate', snapshotsToMigrate.length);
+
+  console.log('Number of versions in the source', fromVersionsRecords.length);
+  console.log('Number of versions in the target', toVersionsRecords.length);
+  console.log('Number of versions to migrate', versionsToMigrate.length);
+
+  const idsMapping = {};
+
+  await Promise.all([
+    rewriteSnapshots(migration.from.snapshots.destination, fromSnapshotsRecordsToRewrite, idsMapping, migration.from.snapshots.logger),
+    rewriteSnapshots(migration.to.snapshots.destination, toSnapshotsRecordsMigrated, idsMapping, migration.to.snapshots.logger),
+  ]);
+
+  await fs.writeFile(path.join(__dirname, 'ids-mapping.json'), JSON.stringify(idsMapping, null, 4));
+
+  console.log('Snapshots migrated\n');
+
+  await Promise.all([
+    rewriteVersions(migration.from.versions.destination, fromVersionsRecordsToRewrite, idsMapping, migration.from.versions.logger),
+    rewriteVersions(migration.to.versions.destination, toVersionsRecordsMigrated, idsMapping, migration.to.versions.logger),
+  ]);
+
+  console.log(`Records treated: ${Object.values(counters).reduce((acc, value) => acc + value, 0)}`);
+  console.log(`⌙ Migrated records: ${counters.migrated}`);
+  console.log(`⌙ Skipped records: ${counters.skipped}`);
+  console.timeEnd('Total time');
+
+  await finalize(migration);
+}());
+
+async function rewriteSnapshots(repository, records, idsMapping, logger) {
+  let i = 1;
+
+  for (const record of records) {
+    const { id: recordId } = await repository.save(record); // eslint-disable-line no-await-in-loop
+
+    idsMapping[record.id] = recordId; // Saves the mapping between the old ID and the new one.
+
+    if (recordId) {
+      logger.info({ message: `Migrated snapshot with new ID: ${recordId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      counters.migrated++;
+    } else {
+      logger.info({ message: 'Skipped snapshot', serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      counters.skipped++;
+    }
+  }
+}
+
+async function rewriteVersions(repository, records, idsMapping, logger) {
+  let i = 1;
+
+  for (const record of records) {
+    const newSnapshotId = idsMapping[record.snapshotId];
+
+    if (!newSnapshotId) {
+      throw new Error(`Snapshot ID ${record.snapshotId} not found for record ${record.id}`);
+    }
+
+    record.snapshotId = newSnapshotId;
+
+    const { id: recordId } = await repository.save(record); // eslint-disable-line no-await-in-loop
+
+    if (recordId) {
+      logger.info({ message: `Migrated version with new ID: ${recordId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      counters.migrated++;
+    } else {
+      logger.info({ message: 'Skipped version', serviceId: record.serviceId, type: record.documentType, id: record.id, current: i++, total: records.length });
+      counters.skipped++;
+    }
+  }
+}
+
+async function initialize(migration) {
+  await Promise.all([
+    migration.from.snapshots.source.initialize(),
+    migration.from.snapshots.destination.initialize(),
+    migration.from.versions.source.initialize(),
+    migration.from.versions.destination.initialize(),
+    migration.to.snapshots.source.initialize(),
+    migration.to.snapshots.destination.initialize(),
+    migration.to.versions.source.initialize(),
+    migration.to.versions.destination.initialize(),
+  ]);
+
+  return Promise.all([
+    importReadmeInGit({ from: migration.from.snapshots.source, to: migration.from.snapshots.destination }),
+    importReadmeInGit({ from: migration.from.versions.source, to: migration.from.versions.destination }),
+    importReadmeInGit({ from: migration.to.snapshots.source, to: migration.to.snapshots.destination }),
+    importReadmeInGit({ from: migration.to.versions.source, to: migration.to.versions.destination }),
+  ]);
+}
+
+async function finalize(migration) {
+  return Promise.all([
+    migration.from.snapshots.source.finalize(),
+    migration.from.snapshots.destination.finalize(),
+    migration.from.versions.source.finalize(),
+    migration.from.versions.destination.finalize(),
+    migration.to.snapshots.source.finalize(),
+    migration.to.snapshots.destination.finalize(),
+    migration.to.versions.source.finalize(),
+    migration.to.versions.destination.finalize(),
+  ]);
+}

package/scripts/history/update-to-full-hash.js

@@ -0,0 +1,61 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import config from 'config';
+
+import GitRepository from '../../src/archivist/recorder/repositories/git/index.js';
+
+import logger from './logger/index.js';
+import { importReadmeInGit } from './utils/index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT_PATH = path.resolve(__dirname, '../../');
+
+(async function main() {
+  console.time('Total time');
+
+  const versionsRepository = new GitRepository({
+    ...config.get('recorder.versions.storage.git'),
+    path: path.resolve(ROOT_PATH, './data/france-elections-versions'),
+  });
+
+  const versionsTargetRepository = new GitRepository({
+    ...config.get('recorder.versions.storage.git'),
+    prefixMessageToSnapshotId: 'This version was recorded after filtering snapshot https://github.com/OpenTermsArchive/france-elections-snapshots/commit/',
+    path: path.resolve(ROOT_PATH, './data/france-elections-versions-hash-updated-test'),
+  });
+
+  const snapshotsRepository = new GitRepository({
+    ...config.get('recorder.snapshots.storage.git'),
+    path: path.resolve(ROOT_PATH, './data/france-elections-snapshots'),
+  });
+
+  await versionsRepository.initialize();
+  await versionsTargetRepository.initialize();
+  await snapshotsRepository.initialize();
+
+  await importReadmeInGit({ from: versionsRepository, to: versionsTargetRepository });
+
+  const total = await versionsRepository.count();
+  let current = 1;
+
+  for await (const record of versionsRepository.iterate()) {
+    const fullSnapshotId = await snapshotsRepository.git.getFullHash(record.snapshotId);
+
+    record.snapshotId = fullSnapshotId;
+
+    const { id: recordId } = await versionsTargetRepository.save(record);
+
+    if (!recordId) {
+      logger.warn({ message: 'Record skipped', serviceId: record.serviceId, type: record.documentType, id: record.id, current, total });
+    } else {
+      logger.info({ message: `Update short sha ${record.snapshotId} to ${fullSnapshotId}`, serviceId: record.serviceId, type: record.documentType, id: record.id, current, total });
+    }
+
+    current++;
+  }
+
+  await versionsRepository.finalize();
+  await versionsTargetRepository.finalize();
+  await snapshotsRepository.finalize();
+}());

package/scripts/history/utils/index.js

@@ -0,0 +1,23 @@
+import fsApi from 'fs';
+
+const fs = fsApi.promises;
+
+export async function importReadmeInGit({ from: sourceRepository, to: targetRepository }) {
+  const sourceRepositoryReadmePath = `${sourceRepository.path}/README.md`;
+  const targetRepositoryReadmePath = `${targetRepository.path}/README.md`;
+
+  const [firstReadmeCommit] = await sourceRepository.git.log(['README.md']);
+
+  if (!firstReadmeCommit) {
+    console.warn(`No commit found for README in ${sourceRepository.path}`);
+
+    return;
+  }
+
+  await fs.copyFile(sourceRepositoryReadmePath, targetRepositoryReadmePath);
+  await targetRepository._commit({
+    filePath: targetRepositoryReadmePath,
+    message: firstReadmeCommit.message,
+    date: firstReadmeCommit.date,
+  });
+}

package/scripts/import/README.md

@@ -0,0 +1,59 @@
+# Import snapshots
+
+Import snapshots history from git to MongoDB.
+
+## How it works
+
+The import process is split into two scripts for performance reasons (reading commits from a huge git repository takes a long time). The first script `loadCommits.js` reads commits from a source repository and inserts them as is (without file contents) into a MongoDB database.
+The second script, `index.js`, reads the commits from the MongoDB database, retrieves the contents from GitHub, applies renaming rules if necessary, and then inserts them into another collection in the database with a format compatible with the OpenTermsArchive application.
+## Configuring
+
+You can change the configuration in `config/import.json`.
+
+```json
+{
+  "import": {
+    "sourcePath": "Path of the source repository",
+    "githubRepository": "Snapshots GitHub repository identifier. Should respect the format: <organisation_or_user_name>/<repository_name>",
+    "mongo": {
+      "connectionURI": "URI for defining connection to the MongoDB instance. See https://docs.mongodb.com/manual/reference/connection-string/",
+      "database": "Database name",
+      "collection": "Collection name"
+    }
+  }
+}
+```
+
+## Adding renaming rules
+
+See the [renamer module documentation](../renamer/README.md).
+
+## Running
+
+**You should execute commands from the `scripts/import` directory to ensure config is properly loaded:**
+
+```
+cd scripts/import
+```
+
+Before importing commits you have to load them in the database:
+```sh
+NODE_ENV=import node scripts/import/loadCommits.js
+```
+
+Then import snapshots from commits:
+```sh
+NODE_ENV=import node scripts/import/index.js
+```
+## Important notes
+
+- Your source repository will be read as is, so checkout the proper branch of commit before running the script.
+
+### Edge cases
+
+The script will:
+
+- Ignore commits which are not a document snapshot (like renaming or documentation commits).
+- Rename document types according to declared rules. See the [renamer module documentation](../renamer/README.md).
+- Rename services according to declared rules. See the [renamer module documentation](../renamer/README.md).
+- Handle duplicates, so you can run it twice without worrying about duplicate entries in the database.

package/scripts/import/config/import.json

@@ -0,0 +1,12 @@
+{
+  "import": {
+    "sourcePath": "../ota-snapshots",
+    "githubRepository": "ambanum/OpenTermsArchive-snapshots",
+    "mongo": {
+      "connectionURI": "mongodb://127.0.0.1:27017",
+      "database": "open-terms-archive-import-retry",
+      "snapshotsCollection": "snapshots",
+      "commitsCollection": "commits"
+    }
+  }
+}

package/scripts/import/index.js

@@ -0,0 +1,224 @@
+import path from 'path';
+import { performance } from 'perf_hooks';
+import { fileURLToPath } from 'url';
+
+import async from 'async';
+import config from 'config';
+import mime from 'mime';
+import { MongoClient } from 'mongodb';
+import nodeFetch from 'node-fetch';
+
+import Git from '../../src/archivist/recorder/repositories/git/git.js';
+import * as renamer from '../utils/renamer/index.js';
+
+import logger from './logger/index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const ROOT_PATH = path.resolve(__dirname, '../../');
+const MAX_PARALLEL = 10;
+const MAX_RETRY = 5;
+const PDF_MIME_TYPE = 'application/pdf';
+const COUNTERS = {
+  imported: 0,
+  skippedNoChanges: 0,
+  errors: 0,
+};
+const commitsNotImported = [];
+
+let sourceRepository;
+let snapshotsCollection;
+let commitsCollection;
+let client;
+
+(async function main() {
+  console.time('Total time');
+  logger.info({ message: 'Start importing…' });
+
+  await initialize();
+
+  const totalToTreat = await commitsCollection.find().count();
+
+  const queue = async.queue(queueWorker, MAX_PARALLEL);
+
+  queue.error(queueErrorHandler);
+  queue.drain(queueDrainHandler(totalToTreat));
+
+  let counter = 1;
+
+  await commitsCollection.find().forEach(commit => {
+    queue.push({ commit, index: counter, total: totalToTreat });
+    counter++;
+  });
+}());
+
+async function initialize() {
+  client = new MongoClient(config.get('import.mongo.connectionURI'));
+
+  await client.connect();
+  const db = client.db(config.get('import.mongo.database'));
+
+  snapshotsCollection = db.collection(config.get('import.mongo.snapshotsCollection'));
+  commitsCollection = db.collection(config.get('import.mongo.commitsCollection'));
+
+  sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('import.sourcePath')) });
+
+  await sourceRepository.initialize();
+  await renamer.loadRules();
+}
+
+async function queueWorker({ commit, index, total }) {
+  return async.retry({
+    times: MAX_RETRY,
+    interval(retryCount) {
+      return 1000 * 2 ** retryCount;
+    },
+  }, callback => {
+    handleCommit(commit, index, total).then(() => {
+      callback();
+    }).catch(error => {
+      callback(error);
+    });
+  });
+}
+
+function queueErrorHandler(error, { commit }) {
+  const [{ file: relativeFilePath }] = commit.diff.files;
+
+  const serviceId = path.dirname(relativeFilePath);
+  const extension = path.extname(relativeFilePath);
+  const documentType = path.basename(relativeFilePath, extension);
+
+  commitsNotImported.push(commit.hash);
+  logger.error({ message: `${error.stack}\nCommit details: ${JSON.stringify(commit, null, 2)}`, serviceId, type: documentType, sha: commit.hash });
+  COUNTERS.errors++;
+}
+
+function queueDrainHandler(totalToTreat) {
+  return () => {
+    const totalTreatedCommits = Object.values(COUNTERS).reduce((acc, value) => acc + value, 0);
+
+    console.log(`\nEntries treated: ${totalTreatedCommits} on ${totalToTreat}`);
+    console.log(`⌙ Entries imported: ${COUNTERS.imported}`);
+    console.log(`⌙ Entries skipped (already on the database): ${COUNTERS.skippedNoChanges}`);
+    console.log(`⌙ Entries with errors: ${COUNTERS.errors}`);
+    console.timeEnd('Total time');
+
+    if (totalTreatedCommits != totalToTreat) {
+      console.error('\n⚠ WARNING: Total treated entries does not match the total number of entries to be treated! ⚠');
+    }
+
+    if (commitsNotImported.length) {
+      console.log('Not imported commits:\n');
+      console.log(commitsNotImported);
+    }
+
+    client.close();
+  };
+}
+
+async function getCommitContent({ sha, serviceId, documentType, extension }) {
+  const start = performance.now();
+  const url = `https://raw.githubusercontent.com/${config.get('import.githubRepository')}/${sha}/${encodeURI(serviceId)}/${encodeURI(documentType)}.${extension}`;
+  const response = await nodeFetch(url);
+  const end = performance.now();
+
+  let content;
+
+  const mimeType = mime.getType(extension);
+
+  if (mimeType == PDF_MIME_TYPE) {
+    content = Buffer.from(await response.arrayBuffer());
+  } else {
+    content = await response.text();
+  }
+
+  if (response.status !== 200) {
+    throw new Error(`Cannot get commit content on Github ${url}. Get HTTP Code ${response.status} and response ${content}`);
+  }
+
+  if (content == '429: Too Many Requests') {
+    throw new TooManyRequestsError(`Cannot get commit content on Github ${url}. 429: Too Many Requests`);
+  }
+
+  logger.info({ message: `Fetched in ${Number(end - start).toFixed(2)} ms`, serviceId, type: documentType, sha });
+
+  return content;
+}
+
+async function handleCommit(commit, index, total) {
+  const [{ file: relativeFilePath }] = commit.diff.files;
+
+  let serviceId = path.dirname(relativeFilePath);
+  const extension = path.extname(relativeFilePath);
+  let documentType = path.basename(relativeFilePath, extension);
+
+  logger.info({
+    message: 'Start to handle commit',
+    serviceId,
+    type: documentType,
+    sha: commit.hash,
+    current: index + 1,
+    total,
+  });
+
+  const alreadyExistsRecord = await snapshotsCollection.findOne({ '_importMetadata.commitSHA': commit.hash });
+
+  if (alreadyExistsRecord) {
+    logger.info({
+      message: 'Skipped commit as an entry already exists for this commit',
+      serviceId,
+      type: documentType,
+      sha: commit.hash,
+    });
+    COUNTERS.skippedNoChanges++;
+
+    return;
+  }
+
+  let content = await getCommitContent({ sha: commit.hash, serviceId, documentType, extension: extension.replace('.', '') });
+
+  ({ serviceId, documentType } = renamer.applyRules(serviceId, documentType));
+
+  const mimeType = mime.getType(extension);
+
+  if (mimeType == PDF_MIME_TYPE) {
+    content = content.toString('utf-8'); // Serialize PDF
+  }
+
+  try {
+    const _importMetadata = { commitSHA: commit.hash };
+
+    if (commit.body.includes('tosback2')) {
+      _importMetadata.provider = 'TOSBack.org';
+      _importMetadata.url = commit.body?.match(/Imported from (.*)\nSnapshot/)[1];
+    }
+
+    const start = performance.now();
+
+    await snapshotsCollection.insertOne({
+      serviceId,
+      documentType,
+      content,
+      mimeType,
+      fetchDate: commit.date,
+      _importMetadata,
+      created_at: new Date(),
+    });
+    const end = performance.now();
+
+    logger.info({ message: `Recorded in ${Number(end - start).toFixed(2)} ms`, serviceId, type: documentType });
+    COUNTERS.imported++;
+  } catch (error) {
+    logger.error({ message: `Not saved: ${commit.date} ${error.stack}`, serviceId, type: documentType });
+    commitsNotImported.push(commit.hash);
+    COUNTERS.errors++;
+  }
+}
+
+export class TooManyRequestsError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'TooManyRequestsError';
+  }
+}