@opentermsarchive/engine 0.15.0

package/scripts/import/loadCommits.js

@@ -0,0 +1,66 @@
+import path from 'path';
+import { performance } from 'perf_hooks';
+import { fileURLToPath } from 'url';
+
+import config from 'config';
+import { MongoClient } from 'mongodb';
+
+import Git from '../../src/archivist/recorder/repositories/git/git.js';
+
+import logger from './logger/index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const ROOT_PATH = path.resolve(__dirname, '../../');
+
+let sourceRepository;
+let collection;
+let client;
+
+(async function main() {
+  console.time('Total time');
+  logger.info({ message: 'Start importing commits…' });
+
+  await initialize();
+
+  logger.info({ message: 'Waiting for git log… (this can take a while)' });
+  const start = performance.now();
+  const commits = (await sourceRepository.log(['--stat=4096'])).sort((a, b) => new Date(a.date) - new Date(b.date));
+  const end = performance.now();
+  const filteredCommits = commits.filter(({ message }) => message.match(/^(Start tracking|Update)/));
+
+  logger.info({ message: `Loaded git log in ${Number((end - start) / 1000).toFixed(2)} s` });
+  logger.info({ message: `Source repo contains ${commits.length} commits` });
+
+  const totalCommitToLoad = filteredCommits.length;
+  const numberOfSkippedCommits = commits.length - totalCommitToLoad;
+
+  if (numberOfSkippedCommits) {
+    logger.info({ message: `Skipped ${numberOfSkippedCommits} commits that do not need to be imported (README, LICENCE…).` });
+  }
+
+  let counter = 1;
+
+  for (const commit of filteredCommits.reverse()) { // reverse array to insert most recent commits first
+    await collection.updateOne({ hash: commit.hash }, { $set: { ...commit } }, { upsert: true }); // eslint-disable-line no-await-in-loop
+
+    if (counter % 1000 == 0) {
+      logger.info({ message: ' ', current: counter, total: totalCommitToLoad });
+    }
+    counter++;
+  }
+
+  await client.close();
+}());
+
+async function initialize() {
+  client = new MongoClient(config.get('import.mongo.connectionURI'));
+
+  await client.connect();
+  const db = client.db(config.get('import.mongo.database'));
+
+  collection = db.collection(config.get('import.mongo.commitsCollection'));
+  sourceRepository = new Git({ path: path.resolve(ROOT_PATH, config.get('import.sourcePath')) });
+
+  await sourceRepository.initialize();
+}

package/scripts/import/logger/index.js

@@ -0,0 +1,43 @@
+import winston from 'winston';
+
+const { combine, timestamp, printf, colorize } = winston.format;
+
+const alignedWithColorsAndTime = combine(
+  colorize(),
+  timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+  printf(({ level, message, timestamp, serviceId, type, sha, current, total }) => {
+    let prefix = ''.padEnd(8);
+
+    if (current && total) {
+      prefix = `${Number(((current) / total) * 100).toFixed(2)}%`.padEnd(8);
+    }
+
+    if (serviceId) {
+      prefix += `${serviceId}`.padEnd(30);
+    }
+
+    if (type) {
+      prefix += `${type}`.padEnd(50);
+    }
+
+    if (sha) {
+      prefix += `${sha}`.padEnd(42);
+    } else {
+      prefix += ''.padEnd(42);
+    }
+
+    return `${timestamp} ${level.padEnd(15)} ${prefix} ${message}`;
+  }),
+);
+
+const consoleTransport = new winston.transports.Console();
+
+const transports = [consoleTransport];
+
+const logger = winston.createLogger({
+  format: alignedWithColorsAndTime,
+  transports,
+  rejectionHandlers: transports,
+});
+
+export default logger;

package/scripts/rewrite/README.md

@@ -0,0 +1,131 @@
+__:warning: These scripts are no longer up-to-date with the codebase and are not guaranteed to work.__
+
+# Rewrite history
+
+As some document types or service names can change over time or as we need to import history from other tools, provided they have an history with the same structure as Open Terms Archive, we need a way to rewrite, reorder and apply changes to the snapshots or versions history.
+
+The script works by reading commits from a **source** repository, applying changes and then committing the result in another, empty or not, **target** repository. So a source repository with commits is required.
+
+When re-writing versions, filters are re-applied on snapshots, so services declarations and history are required.
+
+:warning: Currently, history rewriting only works with Git storage.
+
+## Rewrite snapshots
+
+### Configuring
+
+You can change the **source** and **target** repository in `config/rewrite-snapshots.json`. We use the `recorder` module to write to the **target** repository, so to configure the **target** repo, change the `recorder.snapshots.storage.git.path` value:
+
+```json
+{
+  …
+  "recorder": {
+    "snapshots": {
+      "storage": {
+        "git": {
+          "path": "<Target repository>"
+          …
+        }
+      }
+    }
+  },
+  "rewrite": {
+    "snapshotsSourcePath": "<Source repository>"
+  }
+}
+```
+
+Other configuration elements are inherited from the default `recorder` config.
+
+### Running
+
+Run every command by setting `NODE_ENV` to `rewrite-snapshots`.
+
+Run the script by running:
+
+```sh
+cd scripts/rewrite
+NODE_ENV=rewrite-snapshots node rewrite-snapshots.js
+```
+
+You can write in an empty target repository and initialize it by passing the options `--init`:
+
+```sh
+NODE_ENV=rewrite-snapshots node rewrite-snapshots.js --init
+```
+
+This option will create the repository if it does not exists.
+
+:warning: **If the repository already exist it will be deleted and reinitialized by this options.**
+
+The resulting rewritten history can be found in the configured target repository or by default in the `data/snapshots-rewritten` repository.
+
+## Rewrite versions
+
+### Configuring
+
+You can change the **source** and **target** repository in `config/rewrite-versions.json`. We use the `recorder` module to write to the **target** repository, so to configure the **target** repo, change the `recorder.versions.storage.git.path` value:
+
+```json
+{
+  …
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "path": "<Target repository>"
+          …
+        }
+      }
+    }
+  },
+  "rewrite": {
+    "snapshotsSourcePath": "<Source repository>"
+  }
+}
+```
+
+Other configuration elements are inherited from the default `recorder` config.
+
+### Running
+
+Run every command by setting `NODE_ENV` to `rewrite-versions`.
+
+Run the script by running:
+
+```sh
+cd scripts/rewrite
+NODE_ENV=rewrite-versions node rewrite-versions.js
+```
+
+You can write in an empty target repository and initialize it by passing the options `--init`:
+
+```sh
+NODE_ENV=rewrite-versions node rewrite-versions.js --init
+```
+
+This option will create the repository if it does not exists.
+
+:warning: **If the repository already exist it will be deleted and reinitialized by this options.**
+
+The resulting rewritten history can be found in the configured target repository or by default in the `data/versions-rewritten` repository.
+
+### Important notes
+
+- Your source repository will be read as it, so checkout the proper branch of commit before running the script.
+- If you kill the script during its run, your source repository will probably on a commit in the middle of the history, you need to manually checkout to the proper wanted commit of branche before re-running it.
+
+## Adding renaming rules
+
+See the [renamer module documentation](../renamer/README.md).
+
+### Currently handled cases
+
+Currently, the script will:
+
+- Ignore commits which are not a document snapshot (like renaming or documentation commits)
+- Reorder commits according to their author date
+- Rename document types according to declared rules
+- Rename services according to declared rules
+- Skip commits with empty content
+- Skip commits which do not change the document

package/scripts/rewrite/config/rewrite-snapshots.json

@@ -0,0 +1,32 @@
+{
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "path": "./data/versions",
+          "publish": false,
+          "prefixMessageToSnapshotId": "This version was recorded after filtering snapshot https://github.com/ambanum/OpenTermsArchive-snapshots/commit/",
+          "author": {
+            "name": "Open Terms Archive Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "path": "./data/snapshots-rewritten",
+          "publish": false,
+          "author": {
+            "name": "Open Terms Archive Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        }
+      }
+    }
+  },
+  "rewrite": {
+    "snapshotsSourcePath": "./data/snapshots"
+  }
+}

package/scripts/rewrite/config/rewrite-versions.json

@@ -0,0 +1,32 @@
+{
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "path": "./data/versions-rewritten",
+          "publish": false,
+          "prefixMessageToSnapshotId": "This version was recorded after filtering snapshot https://github.com/ambanum/OpenTermsArchive-snapshots/commit/",
+          "author": {
+            "name": "Open Terms Archive Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "path": "./data/snapshots",
+          "publish": false,
+          "author": {
+            "name": "Open Terms Archive Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        }
+      }
+    }
+  },
+  "rewrite": {
+    "snapshotsSourcePath": "./data/snapshots"
+  }
+}