@twin.org/entity-storage-connector-mongodb 0.0.3-next.2 → 0.0.3-next.20

package/README.md

@@ -1,6 +1,6 @@
-# TWIN Entity Storage Connector MongoDb
+# Entity Storage Connector MongoDB
 
-Entity Storage connector implementation using MongoDb storage.
+This package provides a MongoDB backend for flexible document persistence and evolving schemas. It is designed to work with the wider storage ecosystem so applications can keep behaviour consistent across connectors and environments.
 
 ## Installation
 
@@ -8,18 +8,13 @@ Entity Storage connector implementation using MongoDb storage.
 npm install @twin.org/entity-storage-connector-mongodb
 ```
 
-## Testing
+## Docker
 
-The tests developed are functional tests and need an instance of MongoDb up and running. To run MongoDb locally:
+To perform testing of this component it may be necessary to launch a local instance to communicate with.
 
-```sh
-docker run -p 27500:27017 --name twin-entity-storage-mongodb --hostname mongo -d mongo
-```
-
-Afterwards you can run the tests as follows:
-
-```sh
-npm run test
+```shell
+docker pull mongo:latest
+docker run -d --name twin-entity-storage-mongodb -p 27500:27017 mongo:latest
 ```
 
 ## Examples

package/dist/es/mongoDbEntityStorageConnector.js

@@ -1,8 +1,9 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { BaseError, ComponentFactory, GeneralError, Guards, Is, ObjectHelper } from "@twin.org/core";
-import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, LogicalOperator } from "@twin.org/entity";
+import { BaseError, ComponentFactory, GeneralError, Guards, HealthStatus, Is, ObjectHelper } from "@twin.org/core";
+import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, EntitySchemaPropertyType, LogicalOperator } from "@twin.org/entity";
+import { EntityStorageHelper } from "@twin.org/entity-storage-models";
 import { MongoClient } from "mongodb";
 /**
  * Class for performing entity storage operations using MongoDb.
@@ -22,6 +23,11 @@ export class MongoDbEntityStorageConnector {
      * @internal
      */
     static _PARTITION_KEY = "partitionId";
+    /**
+     * The name for the schema.
+     * @internal
+     */
+    _entitySchemaName;
     /**
      * The schema for the entity.
      * @internal
@@ -53,6 +59,7 @@ export class MongoDbEntityStorageConnector {
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "options.config.host", options.config.host);
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "options.config.database", options.config.database);
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "options.config.collection", options.config.collection);
+        this._entitySchemaName = options.entitySchema;
         this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._partitionContextIds = options.partitionContextIds;
         this._config = options.config;
@@ -113,6 +120,14 @@ export class MongoDbEntityStorageConnector {
         }
         return true;
     }
+    /**
+     * The component needs to be stopped when the node is closed.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns Nothing.
+     */
+    async stop(nodeLoggingComponentType) {
+        await this._client.close();
+    }
     /**
      * Returns the class name of the component.
      * @returns The class name of the component.
@@ -120,6 +135,37 @@ export class MongoDbEntityStorageConnector {
     className() {
         return MongoDbEntityStorageConnector.CLASS_NAME;
     }
+    /**
+     * Returns the health status of the component.
+     * @returns The health status of the component.
+     */
+    async health() {
+        try {
+            await this._client
+                .db(this._config.database)
+                .collection(this._config.collection)
+                .estimatedDocumentCount();
+            return [
+                {
+                    source: MongoDbEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Ok,
+                    description: "healthDescription",
+                    data: { database: this._config.database, collection: this._config.collection }
+                }
+            ];
+        }
+        catch {
+            return [
+                {
+                    source: MongoDbEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Error,
+                    description: "healthDescription",
+                    message: "connectionFailed",
+                    data: { database: this._config.database, collection: this._config.collection }
+                }
+            ];
+        }
+    }
     /**
      * Get the schema for the entities.
      * @returns The schema for the entities.
@@ -154,8 +200,11 @@ export class MongoDbEntityStorageConnector {
             const collection = await this.getCollection();
             const result = await collection.findOne(query);
             ObjectHelper.propertyDelete(result, "_id");
-            ObjectHelper.propertyDelete(result, MongoDbEntityStorageConnector._PARTITION_KEY);
-            return result;
+            return Is.objectValue(result)
+                ? EntityStorageHelper.unPrepareEntity(result, [
+                    MongoDbEntityStorageConnector._PARTITION_KEY
+                ])
+                : undefined;
         }
         catch (err) {
             throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "getFailed", {
@@ -173,15 +222,15 @@ export class MongoDbEntityStorageConnector {
         Guards.object(MongoDbEntityStorageConnector.CLASS_NAME, "entity", entity);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        EntitySchemaHelper.validateEntity(entity, this.getSchema());
+        const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+            ? [{ property: MongoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : undefined, { nullBehavior: "omit" });
         const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
-        const id = entity[primaryKey.property];
+        const id = prepared[primaryKey.property];
         try {
             const filter = { [primaryKey.property]: id };
-            const finalEntity = ObjectHelper.clone(entity);
             if (Is.stringValue(partitionKey)) {
                 filter[MongoDbEntityStorageConnector._PARTITION_KEY] = partitionKey;
-                ObjectHelper.propertySet(finalEntity, MongoDbEntityStorageConnector._PARTITION_KEY, partitionKey);
             }
             if (Is.arrayValue(conditions)) {
                 for (const condition of conditions) {
@@ -189,7 +238,7 @@ export class MongoDbEntityStorageConnector {
                 }
             }
             const collection = await this.getCollection();
-            await collection.findOneAndUpdate(filter, { $set: ObjectHelper.removeEmptyProperties(finalEntity) }, { upsert: true });
+            await collection.findOneAndUpdate(filter, { $set: prepared }, { upsert: true });
         }
         catch (err) {
             throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "setFailed", {
@@ -197,6 +246,62 @@ export class MongoDbEntityStorageConnector {
             }, err);
         }
     }
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    async setBatch(entities) {
+        Guards.arrayValue(MongoDbEntityStorageConnector.CLASS_NAME, "entities", entities);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
+        const preparedEntities = entities.map(entity => EntityStorageHelper.prepareEntity(entity, this._entitySchema, Is.stringValue(partitionKey)
+            ? [{ property: MongoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : undefined, { nullBehavior: "omit" }));
+        try {
+            const collection = await this.getCollection();
+            await collection.bulkWrite(preparedEntities.map(prepared => {
+                const filter = {
+                    [primaryKey.property]: prepared[primaryKey.property]
+                };
+                if (Is.stringValue(partitionKey)) {
+                    filter[MongoDbEntityStorageConnector._PARTITION_KEY] = partitionKey;
+                }
+                return {
+                    updateOne: {
+                        filter,
+                        update: {
+                            $set: prepared
+                        },
+                        upsert: true
+                    }
+                };
+            }));
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "setBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Empty the entity storage.
+     * @returns Nothing.
+     */
+    async empty() {
+        try {
+            const contextIds = await ContextIdStore.getContextIds();
+            const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+            const filter = {};
+            if (Is.stringValue(partitionKey)) {
+                filter[MongoDbEntityStorageConnector._PARTITION_KEY] = partitionKey;
+            }
+            const collection = await this.getCollection();
+            await collection.deleteMany(filter);
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "emptyFailed", undefined, err);
+        }
+    }
     /**
      * Remove the entity.
      * @param id The id of the entity to remove.
@@ -225,6 +330,67 @@ export class MongoDbEntityStorageConnector {
             throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "removeFailed", { id }, err);
         }
     }
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    async removeBatch(ids) {
+        Guards.arrayValue(MongoDbEntityStorageConnector.CLASS_NAME, "ids", ids);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
+            const filter = {
+                [primaryKey.property]: { $in: ids }
+            };
+            if (Is.stringValue(partitionKey)) {
+                filter[MongoDbEntityStorageConnector._PARTITION_KEY] = partitionKey;
+            }
+            const collection = await this.getCollection();
+            await collection.deleteMany(filter);
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "removeBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Teardown the entity storage by dropping the collection.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the teardown process was successful.
+     */
+    async teardown(nodeLoggingComponentType) {
+        const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        await nodeLogging?.log({
+            level: "info",
+            source: MongoDbEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "collectionDropping",
+            data: { collection: this._config.collection }
+        });
+        try {
+            const collection = await this.getCollection();
+            await collection.drop();
+            await nodeLogging?.log({
+                level: "info",
+                source: MongoDbEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "collectionDropped",
+                data: { collection: this._config.collection }
+            });
+            return true;
+        }
+        catch (err) {
+            await nodeLogging?.log({
+                level: "error",
+                source: MongoDbEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "teardownFailed",
+                error: BaseError.fromError(err)
+            });
+            return false;
+        }
+    }
     /**
      * Find all the entities which match the conditions.
      * @param conditions The conditions to match for the entities.
@@ -239,24 +405,7 @@ export class MongoDbEntityStorageConnector {
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
         const returnSize = limit ?? MongoDbEntityStorageConnector._DEFAULT_LIMIT;
-        const finalConditions = {
-            conditions: [],
-            logicalOperator: LogicalOperator.And
-        };
-        if (Is.stringValue(partitionKey)) {
-            finalConditions.conditions.push({
-                property: MongoDbEntityStorageConnector._PARTITION_KEY,
-                comparison: ComparisonOperator.Equals,
-                value: partitionKey
-            });
-        }
-        if (!Is.empty(conditions)) {
-            finalConditions.conditions.push(conditions);
-        }
-        const filter = {};
-        if (finalConditions.conditions.length > 0) {
-            this.buildQueryParameters("", finalConditions, filter);
-        }
+        const filter = this.buildFilter(conditions, partitionKey);
         const sort = new Map();
         if (Array.isArray(sortProperties)) {
             for (const sortProperty of sortProperties) {
@@ -277,30 +426,117 @@ export class MongoDbEntityStorageConnector {
             ?.find(filter, { projection })
             .sort(sort)
             .skip(cursorValue)
-            .limit(returnSize)
+            .limit(returnSize + 1)
             .toArray();
-        const entities = entitiesResult ?? [];
-        for (const entity of entities) {
+        const rawResults = entitiesResult ?? [];
+        const hasMore = rawResults.length > returnSize;
+        const entities = hasMore ? rawResults.slice(0, returnSize) : rawResults;
+        for (let i = 0; i < entities.length; i++) {
+            const entity = entities[i];
             ObjectHelper.propertyDelete(entity, "_id");
-            ObjectHelper.propertyDelete(entity, MongoDbEntityStorageConnector._PARTITION_KEY);
+            entities[i] = EntityStorageHelper.unPrepareEntity(entity, [
+                MongoDbEntityStorageConnector._PARTITION_KEY
+            ]);
         }
         return {
             entities,
-            cursor: entities?.length === returnSize ? String(cursorValue + returnSize) : undefined
+            cursor: hasMore ? String(cursorValue + returnSize) : undefined
         };
     }
     /**
-     * Drop the collection.
-     * @returns Nothing.
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
+     */
+    async count(conditions) {
+        try {
+            const contextIds = await ContextIdStore.getContextIds();
+            const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+            const filter = this.buildFilter(conditions, partitionKey);
+            return await this._client
+                .db(this._config.database)
+                .collection(this._config.collection)
+                .countDocuments(filter);
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "countFailed", undefined, err);
+        }
+    }
+    /**
+     * Get all unique partition context ids present in the collection.
+     * @returns An array of context id objects, one per unique partition.
      */
-    async collectionDrop() {
+    async getPartitionContextIds() {
+        if (!Is.arrayValue(this._partitionContextIds)) {
+            return [];
+        }
         try {
             const collection = await this.getCollection();
-            await collection.drop();
+            const partitionIds = await collection.distinct(MongoDbEntityStorageConnector._PARTITION_KEY);
+            return partitionIds
+                .filter((id) => Is.stringValue(id))
+                .map(partitionId => ContextIdHelper.shortSplit(this._partitionContextIds ?? [], partitionId));
         }
-        catch {
-            // Ignore errors
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "getPartitionContextIdsFailed", undefined, err);
+        }
+    }
+    /**
+     * Create the target connector for performing the migration using a temporary collection.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    async createTargetConnector(newEntitySchema) {
+        const migrationCollectionName = `${this._config.collection}Migration${Date.now()}`;
+        return new MongoDbEntityStorageConnector({
+            entitySchema: newEntitySchema,
+            config: {
+                ...this._config,
+                collection: migrationCollectionName
+            },
+            partitionContextIds: this._partitionContextIds
+        });
+    }
+    /**
+     * Finalize the migration by dropping the source collection and renaming the migration collection to the original name.
+     * @param targetConnector The connector holding the migrated data in a temporary collection.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The logging component type to use during finalization.
+     * @returns The final connector using the original collection name with the new schema.
+     */
+    async finalizeMigration(targetConnector, options, loggingComponentType) {
+        // Only rename if the migration collection was actually created (it won't exist if no
+        // entities were written, since MongoDB creates collections lazily on first write).
+        const migrationCollection = await targetConnector.getCollection();
+        const collections = await targetConnector._client
+            .db(targetConnector._config.database)
+            .listCollections({ name: targetConnector._config.collection })
+            .toArray();
+        if (collections.length > 0) {
+            // Teardown the existing table with the original name to free up the name for the new table
+            await this.teardown(loggingComponentType);
+            await migrationCollection.rename(this._config.collection);
         }
+        const finalConnector = new MongoDbEntityStorageConnector({
+            entitySchema: targetConnector._entitySchemaName,
+            config: this._config,
+            partitionContextIds: this._partitionContextIds
+        });
+        if (await finalConnector.bootstrap(loggingComponentType)) {
+            await targetConnector.stop?.();
+            return finalConnector;
+        }
+        throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "finalizeMigrationFailedBootstrap", undefined);
+    }
+    /**
+     * Cleanup a failed or aborted migration by dropping the temporary migration collection.
+     * @param targetConnector The target connector to cleanup.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging.
+     * @returns A promise that resolves when the cleanup is complete.
+     */
+    async cleanupMigration(targetConnector, options, loggingComponentType) {
+        await targetConnector?.teardown?.(loggingComponentType);
     }
     /**
      * Create a new DB connection configuration.
@@ -324,6 +560,34 @@ export class MongoDbEntityStorageConnector {
         const { database, collection } = this._config;
         return this._client.db(database).collection(collection);
     }
+    /**
+     * Build a MongoDB filter combining partition key and optional conditions.
+     * @param conditions The optional entity conditions to include.
+     * @param partitionKey The partition key value.
+     * @returns The MongoDB filter object.
+     * @internal
+     */
+    buildFilter(conditions, partitionKey) {
+        const finalConditions = {
+            conditions: [],
+            logicalOperator: LogicalOperator.And
+        };
+        if (Is.stringValue(partitionKey)) {
+            finalConditions.conditions.push({
+                property: MongoDbEntityStorageConnector._PARTITION_KEY,
+                comparison: ComparisonOperator.Equals,
+                value: partitionKey
+            });
+        }
+        if (!Is.empty(conditions)) {
+            finalConditions.conditions.push(conditions);
+        }
+        const filter = {};
+        if (finalConditions.conditions.length > 0) {
+            this.buildQueryParameters("", finalConditions, filter);
+        }
+        return filter;
+    }
     /**
      * Create an MongoDB filter query.
      * @param objectPath The path for the nested object.
@@ -352,8 +616,17 @@ export class MongoDbEntityStorageConnector {
             }
         }
         else {
-            const prop = objectPath ? `${objectPath}.${condition.property}` : String(condition.property);
-            const comparison = this.mapComparisonOperator(condition.comparison, condition.value);
+            const propertyPath = String(condition.property);
+            const prop = objectPath ? `${objectPath}.${propertyPath}` : propertyPath;
+            const propertyParts = propertyPath.split(".");
+            const schemaLookupName = propertyParts.length > 1 ? propertyParts[0] : propertyPath;
+            const propertySchema = this._entitySchema.properties?.find(p => p.property === schemaLookupName);
+            // For dot-notation paths the leaf field is always a string value; using the root
+            // type directly would send Includes into $elemMatch which does not work for nested
+            // string fields. Keeping String here causes mapComparisonOperator to emit $regex,
+            // which MongoDB handles correctly for both nested object and array traversal.
+            const propertyType = propertyParts.length > 1 ? EntitySchemaPropertyType.String : propertySchema?.type;
+            const comparison = this.mapComparisonOperator(condition.comparison, condition.value, propertyType);
             filter[prop] = comparison;
         }
     }
@@ -361,10 +634,11 @@ export class MongoDbEntityStorageConnector {
      * Map the framework comparison operators to those in MongoDB.
      * @param comparison The comparison operator.
      * @param value The value to compare.
+     * @param type The type of the property from the schema.
      * @returns The MongoDB comparison expression.
      * @internal
      */
-    mapComparisonOperator(comparison, value) {
+    mapComparisonOperator(comparison, value, type) {
         switch (comparison) {
             case ComparisonOperator.Equals:
                 return value;
@@ -381,8 +655,25 @@ export class MongoDbEntityStorageConnector {
             case ComparisonOperator.In:
                 return { $in: Array.isArray(value) ? value : [value] };
             case ComparisonOperator.Includes:
+                // For string fields, use regex for substring matching
+                if (type === EntitySchemaPropertyType.String) {
+                    // Escape special regex characters in the value
+                    const escapedValue = String(value).replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
+                    return { $regex: escapedValue };
+                }
+                // For array and object fields, use $elemMatch
+                if (type === EntitySchemaPropertyType.Array || type === EntitySchemaPropertyType.Object) {
+                    return { $elemMatch: { $eq: value } };
+                }
+                // Fallback to $elemMatch for backwards compatibility
                 return { $elemMatch: { $eq: value } };
             case ComparisonOperator.NotIncludes:
+                // For string fields, use negated regex
+                if (type === EntitySchemaPropertyType.String) {
+                    const escapedValue = String(value).replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
+                    return { $not: { $regex: escapedValue } };
+                }
+                // For arrays, use $elemMatch with $ne
                 return { $elemMatch: { $ne: value } };
             default:
                 throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "unsupportedComparisonOperator", { comparison });