@opentermsarchive/engine 0.15.0

package/src/archivist/recorder/repositories/interface.js

@@ -0,0 +1,108 @@
+/**
+ * Interface for classes that model a collection of domain objects with querying capabilities
+ * @see {@link https://martinfowler.com/eaaCatalog/repository.html|Repository}
+ * @interface
+ */
+export default class RepositoryInterface {
+  /**
+  * [Optional] Initialize repository
+  * Override this method if the repository needs some asynchronous initialization code (open database connection and create collections, initialize Git…)
+  *
+  * @returns {Promise<Repository>} Promise that will be resolved with the current repository instance
+  */
+  async initialize() {
+    return this;
+  }
+
+  /**
+  * [Optional] Finalize repository
+  * Override this method if the repository needs some asynchronous code to properly close the repository (close database connection, push changes on Git remote…)
+  *
+  * @returns {Promise<Repository>} Promise that will be resolved with the current repository instance
+  */
+  async finalize() {
+    return this;
+  }
+
+  /**
+  * Persist the given record if it does not already exist in repository
+  *
+  * @param {Record} record - Record to persist
+  * @returns {Promise<Record>} Promise that will be resolved with the given record when it has been persisted
+  */
+  async save(record) {
+    throw new Error(`#save method is not implemented in ${this.constructor.name}`);
+  }
+
+  /**
+  * Find the most recent record that matches the given service ID and document type and optionally the page ID
+  * In case of snapshots, if the record is related to a multipage document, the page ID is required to find the corresponding snapshot
+  *
+  * @param {string} serviceId - Service ID of record to find
+  * @param {string} documentType - Document type of record to find
+  * @param {string} [pageId] - Page ID of record to find. Used to differentiate pages of multipage document. Not necessary for single page document
+  * @returns {Promise<Record>} Promise that will be resolved with the found record or an empty object if none match the given criteria
+  */
+  async findLatest(serviceId, documentType, pageId) {
+    throw new Error(`#findLatest method is not implemented in ${this.constructor.name}`);
+  }
+
+  /**
+  * Find the record that matches the given record ID
+  *
+  * @param {string} recordId - Record ID of the record to find
+  * @returns {Promise<Record>} Promise that will be resolved with the found record or an empty object if none match the given ID
+  */
+  async findById(recordId) {
+    throw new Error(`#findById method is not implemented in ${this.constructor.name}`);
+  }
+
+  /**
+  * Find all records
+  * For performance reasons, the content of the records will not be loaded by default. Use #loadRecordContent to load the content of individual records
+  *
+  * @see RepositoryInterface#loadRecordContent
+  * @returns {Promise<Array<Record>>} Promise that will be resolved with an array of all records
+  */
+  async findAll() {
+    throw new Error(`#findAll method is not implemented in ${this.constructor.name}`);
+  }
+
+  /**
+  * Count the total number of records in the repository
+  * For performance reasons, use this method rather than counting the number of entries returned by #findAll if you only need the size of a repository
+  *
+  * @returns {Promise<number>} Promise that will be resolved with the total number of records
+  */
+  async count() {
+    throw new Error(`#count method is not implemented in ${this.constructor.name}`);
+  }
+
+  /**
+  * Iterate over all records in the repository, from oldest to most recent
+  *
+  * @yields {Record}
+  */
+  async* iterate() {
+    throw new Error(`#iterate method is not implemented in ${this.constructor.name}`);
+  }
+
+  /**
+  * Remove all records
+  *
+  * @returns {Promise} Promise that will be resolved when all records are removed
+  */
+  async removeAll() {
+    throw new Error(`#removeAll method is not implemented in ${this.constructor.name}`);
+  }
+
+  /**
+  * Load content of the given record
+  *
+  * @param {Record} record - Record of which to populate content
+  * @returns {Promise<Record>} Promise that will be resolved with the given record when its content has been loaded
+  */
+  async loadRecordContent(record) {
+    throw new Error(`#loadRecordContent method is not implemented in ${this.constructor.name}`);
+  }
+}

package/src/archivist/recorder/repositories/mongo/dataMapper.js

@@ -0,0 +1,32 @@
+import { ObjectId } from 'mongodb';
+
+import Record from '../../record.js';
+
+export function toPersistence(record) {
+  const documentFields = Object.fromEntries(Object.entries(record));
+
+  if (documentFields.snapshotIds) {
+    documentFields.snapshotIds = record.snapshotIds.map(snapshotId => new ObjectId(snapshotId));
+  }
+
+  documentFields.content = record.content;
+  documentFields.created_at = new Date();
+
+  return documentFields;
+}
+
+export function toDomain(document) {
+  const { _id, serviceId, documentType, pageId, fetchDate, mimeType, isRefilter, isFirstRecord, snapshotIds } = document;
+
+  return new Record({
+    id: _id.toString(),
+    serviceId,
+    documentType,
+    pageId,
+    mimeType,
+    fetchDate: new Date(fetchDate),
+    isFirstRecord: Boolean(isFirstRecord),
+    isRefilter: Boolean(isRefilter),
+    snapshotIds: snapshotIds?.map(snapshotId => snapshotId.toString()) || [],
+  });
+}

package/src/archivist/recorder/repositories/mongo/index.js

@@ -0,0 +1,121 @@
+/**
+ * This module is the boundary beyond which the usage of MongoDB is abstracted.
+ * Object IDs are used as opaque unique IDs.
+ */
+
+import { MongoClient, ObjectId, Binary } from 'mongodb';
+
+import RepositoryInterface from '../interface.js';
+
+import * as DataMapper from './dataMapper.js';
+
+export default class MongoRepository extends RepositoryInterface {
+  constructor({ database: databaseName, collection: collectionName, connectionURI }) {
+    super();
+
+    this.client = new MongoClient(connectionURI);
+    this.databaseName = databaseName;
+    this.collectionName = collectionName;
+  }
+
+  async initialize() {
+    await this.client.connect();
+    const db = this.client.db(this.databaseName);
+
+    this.collection = db.collection(this.collectionName);
+
+    return this;
+  }
+
+  async finalize() {
+    return this.client.close();
+  }
+
+  async save(record) {
+    const { serviceId, documentType } = record;
+
+    if (record.isFirstRecord === undefined || record.isFirstRecord === null) {
+      record.isFirstRecord = !await this.collection.findOne({ serviceId, documentType });
+    }
+
+    const documentFields = await this.#toPersistence(record);
+    const previousRecord = await this.findLatest(serviceId, documentType);
+
+    if (previousRecord?.content == documentFields.content) {
+      return Object(null);
+    }
+
+    const insertResult = await this.collection.insertOne(documentFields);
+
+    record.id = insertResult.insertedId.toString();
+
+    return record;
+  }
+
+  async findLatest(serviceId, documentType) {
+    const [mongoDocument] = await this.collection.find({ serviceId, documentType }).limit(1).sort({ fetchDate: -1 }).toArray(); // `findOne` doesn't support the `sort` method, so even for only one document use `find`
+
+    return this.#toDomain(mongoDocument);
+  }
+
+  async findById(recordId) {
+    const mongoDocument = await this.collection.findOne({ _id: new ObjectId(recordId) });
+
+    return this.#toDomain(mongoDocument);
+  }
+
+  async findAll() {
+    return Promise.all((await this.collection.find().project({ content: 0 }).sort({ fetchDate: 1 }).toArray())
+      .map(mongoDocument => this.#toDomain(mongoDocument, { deferContentLoading: true })));
+  }
+
+  async count() {
+    return this.collection.find().count();
+  }
+
+  async* iterate() {
+    const cursor = this.collection.find().sort({ fetchDate: 1 });
+
+    /* eslint-disable no-await-in-loop */
+    while (await cursor.hasNext()) {
+      const mongoDocument = await cursor.next();
+
+      yield this.#toDomain(mongoDocument);
+    }
+    /* eslint-enable no-await-in-loop */
+  }
+
+  async removeAll() {
+    return this.collection.deleteMany();
+  }
+
+  async loadRecordContent(record) {
+    const { content } = await this.collection.findOne({ _id: new ObjectId(record.id) }, { projection: { content: 1 } });
+
+    record.content = content instanceof Binary ? content.buffer : content;
+  }
+
+  async #toDomain(document, { deferContentLoading } = {}) {
+    if (!document) {
+      return null;
+    }
+
+    const record = DataMapper.toDomain(document);
+
+    if (deferContentLoading) {
+      return record;
+    }
+
+    await this.loadRecordContent(record);
+
+    return record;
+  }
+
+  async #toPersistence(record) {
+    if (record.content === undefined || record.content === null) {
+      await this.repository.loadRecordContent(record);
+    }
+
+    return DataMapper.toPersistence(record);
+  }
+}