@twin.org/entity-storage-connector-mongodb 0.0.3-next.3 → 0.0.3-next.30

package/README.md

@@ -1,6 +1,6 @@
-# TWIN Entity Storage Connector MongoDb
+# Entity Storage Connector MongoDB
 
-Entity Storage connector implementation using MongoDb storage.
+This package provides a MongoDB backend for flexible document persistence and evolving schemas. It is designed to work with the wider storage ecosystem so applications can keep behaviour consistent across connectors and environments.
 
 ## Installation
 
@@ -8,18 +8,13 @@ Entity Storage connector implementation using MongoDb storage.
 npm install @twin.org/entity-storage-connector-mongodb
 ```
 
-## Testing
+## Docker
 
-The tests developed are functional tests and need an instance of MongoDb up and running. To run MongoDb locally:
+To perform testing of this component it may be necessary to launch a local instance to communicate with.
 
-```sh
-docker run -p 27500:27017 --name twin-entity-storage-mongodb --hostname mongo -d mongo
-```
-
-Afterwards you can run the tests as follows:
-
-```sh
-npm run test
+```shell
+docker pull mongo:latest
+docker run -d --name twin-entity-storage-mongodb -p 27500:27017 mongo:latest
 ```
 
 ## Examples

package/dist/es/models/IMongoDbEntityStorageConnectorConstructorOptions.js.map

@@ -1 +1 @@
-{"version":3,"file":"IMongoDbEntityStorageConnectorConstructorOptions.js","sourceRoot":"","sources":["../../../src/models/IMongoDbEntityStorageConnectorConstructorOptions.ts"],"names":[],"mappings":"","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport type { IMongoDbEntityStorageConnectorConfig } from \"./IMongoDbEntityStorageConnectorConfig.js\";\n\n/**\n * The options for the MongoDb entity storage connector constructor.\n */\nexport interface IMongoDbEntityStorageConnectorConstructorOptions {\n\t/**\n\t * The schema for the entity.\n\t */\n\tentitySchema: string;\n\n\t/**\n\t * The keys to use from the context ids to create partitions.\n\t */\n\tpartitionContextIds?: string[];\n\n\t/**\n\t * The type of logging component to use.\n\t * @default logging\n\t */\n\tloggingComponentType?: string;\n\n\t/**\n\t * The configuration for the connector.\n\t */\n\tconfig: IMongoDbEntityStorageConnectorConfig;\n}\n"]}
+{"version":3,"file":"IMongoDbEntityStorageConnectorConstructorOptions.js","sourceRoot":"","sources":["../../../src/models/IMongoDbEntityStorageConnectorConstructorOptions.ts"],"names":[],"mappings":"","sourcesContent":["// Copyright 2024 IOTA Stiftung.\n// SPDX-License-Identifier: Apache-2.0.\nimport type { IMongoDbEntityStorageConnectorConfig } from \"./IMongoDbEntityStorageConnectorConfig.js\";\n\n/**\n * The options for the MongoDb entity storage connector constructor.\n */\nexport interface IMongoDbEntityStorageConnectorConstructorOptions {\n\t/**\n\t * The schema for the entity.\n\t */\n\tentitySchema: string;\n\n\t/**\n\t * The keys to use from the context ids to create partitions.\n\t */\n\tpartitionContextIds?: string[];\n\n\t/**\n\t * The type of logging component to use.\n\t */\n\tloggingComponentType?: string;\n\n\t/**\n\t * The configuration for the connector.\n\t */\n\tconfig: IMongoDbEntityStorageConnectorConfig;\n}\n"]}

package/dist/es/mongoDbEntityStorageConnector.js

@@ -1,8 +1,9 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { BaseError, ComponentFactory, GeneralError, Guards, Is, ObjectHelper } from "@twin.org/core";
-import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, LogicalOperator } from "@twin.org/entity";
+import { BaseError, ComponentFactory, GeneralError, Guards, HealthStatus, Is, ObjectHelper, Validation } from "@twin.org/core";
+import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, EntitySchemaPropertyType, LogicalOperator } from "@twin.org/entity";
+import { EntityStorageHelper } from "@twin.org/entity-storage-models";
 import { MongoClient } from "mongodb";
 /**
  * Class for performing entity storage operations using MongoDb.
@@ -18,10 +19,10 @@ export class MongoDbEntityStorageConnector {
      */
     static _DEFAULT_LIMIT = 40;
     /**
-     * Partition id field name.
+     * The name for the schema.
      * @internal
      */
-    static _PARTITION_KEY = "partitionId";
+    _entitySchemaName;
     /**
      * The schema for the entity.
      * @internal
@@ -53,6 +54,7 @@ export class MongoDbEntityStorageConnector {
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "options.config.host", options.config.host);
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "options.config.database", options.config.database);
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "options.config.collection", options.config.collection);
+        this._entitySchemaName = options.entitySchema;
         this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._partitionContextIds = options.partitionContextIds;
         this._config = options.config;
@@ -113,6 +115,14 @@ export class MongoDbEntityStorageConnector {
         }
         return true;
     }
+    /**
+     * The component needs to be stopped when the node is closed.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns Nothing.
+     */
+    async stop(nodeLoggingComponentType) {
+        await this._client.close();
+    }
     /**
      * Returns the class name of the component.
      * @returns The class name of the component.
@@ -120,6 +130,37 @@ export class MongoDbEntityStorageConnector {
     className() {
         return MongoDbEntityStorageConnector.CLASS_NAME;
     }
+    /**
+     * Returns the health status of the component.
+     * @returns The health status of the component.
+     */
+    async health() {
+        try {
+            await this._client
+                .db(this._config.database)
+                .collection(this._config.collection)
+                .estimatedDocumentCount();
+            return [
+                {
+                    source: MongoDbEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Ok,
+                    description: "healthDescription",
+                    data: { database: this._config.database, collection: this._config.collection }
+                }
+            ];
+        }
+        catch {
+            return [
+                {
+                    source: MongoDbEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Error,
+                    description: "healthDescription",
+                    message: "connectionFailed",
+                    data: { database: this._config.database, collection: this._config.collection }
+                }
+            ];
+        }
+    }
     /**
      * Get the schema for the entities.
      * @returns The schema for the entities.
@@ -136,16 +177,11 @@ export class MongoDbEntityStorageConnector {
      */
     async get(id, secondaryIndex, conditions) {
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "id", id);
-        const contextIds = await ContextIdStore.getContextIds();
-        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
         try {
             const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
             const query = Is.empty(secondaryIndex)
                 ? { [primaryKey.property]: id }
                 : { [secondaryIndex]: id };
-            if (Is.stringValue(partitionKey)) {
-                query[MongoDbEntityStorageConnector._PARTITION_KEY] = partitionKey;
-            }
             if (conditions) {
                 for (const condition of conditions) {
                     query[condition.property] = condition.value;
@@ -154,8 +190,9 @@ export class MongoDbEntityStorageConnector {
             const collection = await this.getCollection();
             const result = await collection.findOne(query);
             ObjectHelper.propertyDelete(result, "_id");
-            ObjectHelper.propertyDelete(result, MongoDbEntityStorageConnector._PARTITION_KEY);
-            return result;
+            return Is.objectValue(result)
+                ? EntityStorageHelper.unPrepareEntity(result, [])
+                : undefined;
         }
         catch (err) {
             throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "getFailed", {
@@ -171,25 +208,20 @@ export class MongoDbEntityStorageConnector {
      */
     async set(entity, conditions) {
         Guards.object(MongoDbEntityStorageConnector.CLASS_NAME, "entity", entity);
-        const contextIds = await ContextIdStore.getContextIds();
-        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        EntitySchemaHelper.validateEntity(entity, this.getSchema());
+        const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, undefined, {
+            nullBehavior: "omit"
+        });
         const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
-        const id = entity[primaryKey.property];
+        const id = prepared[primaryKey.property];
         try {
             const filter = { [primaryKey.property]: id };
-            const finalEntity = ObjectHelper.clone(entity);
-            if (Is.stringValue(partitionKey)) {
-                filter[MongoDbEntityStorageConnector._PARTITION_KEY] = partitionKey;
-                ObjectHelper.propertySet(finalEntity, MongoDbEntityStorageConnector._PARTITION_KEY, partitionKey);
-            }
             if (Is.arrayValue(conditions)) {
                 for (const condition of conditions) {
                     filter[condition.property] = condition.value;
                 }
             }
             const collection = await this.getCollection();
-            await collection.findOneAndUpdate(filter, { $set: ObjectHelper.removeEmptyProperties(finalEntity) }, { upsert: true });
+            await collection.findOneAndUpdate(filter, { $set: prepared }, { upsert: true });
         }
         catch (err) {
             throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "setFailed", {
@@ -197,6 +229,51 @@ export class MongoDbEntityStorageConnector {
             }, err);
         }
     }
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    async setBatch(entities) {
+        Guards.arrayValue(MongoDbEntityStorageConnector.CLASS_NAME, "entities", entities);
+        const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
+        const preparedEntities = entities.map(entity => EntityStorageHelper.prepareEntity(entity, this._entitySchema, undefined, {
+            nullBehavior: "omit"
+        }));
+        try {
+            const collection = await this.getCollection();
+            await collection.bulkWrite(preparedEntities.map(prepared => {
+                const filter = {
+                    [primaryKey.property]: prepared[primaryKey.property]
+                };
+                return {
+                    updateOne: {
+                        filter,
+                        update: {
+                            $set: prepared
+                        },
+                        upsert: true
+                    }
+                };
+            }));
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "setBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Empty the entity storage.
+     * @returns Nothing.
+     */
+    async empty() {
+        try {
+            const collection = await this.getCollection();
+            await collection.deleteMany({});
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "emptyFailed", undefined, err);
+        }
+    }
     /**
      * Remove the entity.
      * @param id The id of the entity to remove.
@@ -205,14 +282,9 @@ export class MongoDbEntityStorageConnector {
      */
     async remove(id, conditions) {
         Guards.stringValue(MongoDbEntityStorageConnector.CLASS_NAME, "id", id);
-        const contextIds = await ContextIdStore.getContextIds();
-        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
         try {
             const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
             const query = { [primaryKey.property]: id };
-            if (Is.stringValue(partitionKey)) {
-                query[MongoDbEntityStorageConnector._PARTITION_KEY] = partitionKey;
-            }
             if (conditions) {
                 for (const condition of conditions) {
                     query[condition.property] = condition.value;
@@ -225,6 +297,62 @@ export class MongoDbEntityStorageConnector {
             throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "removeFailed", { id }, err);
         }
     }
+    /**
+     * Remove multiple entities by id.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    async removeBatch(ids) {
+        Guards.arrayValue(MongoDbEntityStorageConnector.CLASS_NAME, "ids", ids);
+        try {
+            const primaryKey = EntitySchemaHelper.getPrimaryKey(this.getSchema());
+            const filter = {
+                [primaryKey.property]: { $in: ids }
+            };
+            const collection = await this.getCollection();
+            await collection.deleteMany(filter);
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "removeBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Teardown the entity storage by dropping the collection.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the teardown process was successful.
+     */
+    async teardown(nodeLoggingComponentType) {
+        const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        await nodeLogging?.log({
+            level: "info",
+            source: MongoDbEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "collectionDropping",
+            data: { collection: this._config.collection }
+        });
+        try {
+            const collection = await this.getCollection();
+            await collection.drop();
+            await nodeLogging?.log({
+                level: "info",
+                source: MongoDbEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "collectionDropped",
+                data: { collection: this._config.collection }
+            });
+            return true;
+        }
+        catch (err) {
+            await nodeLogging?.log({
+                level: "error",
+                source: MongoDbEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "teardownFailed",
+                error: BaseError.fromError(err)
+            });
+            return false;
+        }
+    }
     /**
      * Find all the entities which match the conditions.
      * @param conditions The conditions to match for the entities.
@@ -236,27 +364,15 @@ export class MongoDbEntityStorageConnector {
      * and a cursor which can be used to request more entities.
      */
     async query(conditions, sortProperties, properties, cursor, limit) {
-        const contextIds = await ContextIdStore.getContextIds();
-        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        const returnSize = limit ?? MongoDbEntityStorageConnector._DEFAULT_LIMIT;
-        const finalConditions = {
-            conditions: [],
-            logicalOperator: LogicalOperator.And
-        };
-        if (Is.stringValue(partitionKey)) {
-            finalConditions.conditions.push({
-                property: MongoDbEntityStorageConnector._PARTITION_KEY,
-                comparison: ComparisonOperator.Equals,
-                value: partitionKey
-            });
-        }
-        if (!Is.empty(conditions)) {
-            finalConditions.conditions.push(conditions);
-        }
-        const filter = {};
-        if (finalConditions.conditions.length > 0) {
-            this.buildQueryParameters("", finalConditions, filter);
+        EntityStorageHelper.validateSortProperties(this._entitySchema, sortProperties);
+        EntityStorageHelper.validateProperties(this._entitySchema, properties);
+        if (!Is.empty(limit)) {
+            const validationFailures = [];
+            Validation.integer("limit", limit, validationFailures, undefined, { minValue: 1 });
+            Validation.asValidationError(MongoDbEntityStorageConnector.CLASS_NAME, "query", validationFailures);
         }
+        const returnSize = limit ?? MongoDbEntityStorageConnector._DEFAULT_LIMIT;
+        const filter = this.buildFilter(conditions);
         const sort = new Map();
         if (Array.isArray(sortProperties)) {
             for (const sortProperty of sortProperties) {
@@ -277,31 +393,123 @@ export class MongoDbEntityStorageConnector {
             ?.find(filter, { projection })
             .sort(sort)
             .skip(cursorValue)
-            .limit(returnSize)
+            .limit(returnSize + 1)
             .toArray();
-        const entities = entitiesResult ?? [];
-        for (const entity of entities) {
+        const rawResults = entitiesResult ?? [];
+        const hasMore = rawResults.length > returnSize;
+        const entities = hasMore ? rawResults.slice(0, returnSize) : rawResults;
+        for (let i = 0; i < entities.length; i++) {
+            const entity = entities[i];
             ObjectHelper.propertyDelete(entity, "_id");
-            ObjectHelper.propertyDelete(entity, MongoDbEntityStorageConnector._PARTITION_KEY);
+            entities[i] = EntityStorageHelper.unPrepareEntity(entity, []);
         }
         return {
             entities,
-            cursor: entities?.length === returnSize ? String(cursorValue + returnSize) : undefined
+            cursor: hasMore ? String(cursorValue + returnSize) : undefined
         };
     }
     /**
-     * Drop the collection.
-     * @returns Nothing.
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
      */
-    async collectionDrop() {
+    async count(conditions) {
         try {
+            const filter = this.buildFilter(conditions);
             const collection = await this.getCollection();
-            await collection.drop();
+            return await collection.countDocuments(filter);
         }
-        catch {
-            // Ignore errors
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "countFailed", undefined, err);
+        }
+    }
+    /**
+     * Get all unique partition context ids present in the collection.
+     * @returns An array of context id objects, one per unique partition.
+     */
+    async getPartitionContextIds() {
+        if (!Is.arrayValue(this._partitionContextIds)) {
+            return [];
+        }
+        try {
+            const prefix = `${this._config.collection}_`;
+            const escapedPrefix = prefix.replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
+            const db = this._client.db(this._config.database);
+            const collections = await db
+                .listCollections({ name: { $regex: `^${escapedPrefix}` } })
+                .toArray();
+            return collections.map(col => ContextIdHelper.shortSplit(this._partitionContextIds ?? [], col.name.slice(prefix.length)));
+        }
+        catch (err) {
+            throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "getPartitionContextIdsFailed", undefined, err);
         }
     }
+    /**
+     * Create the target connector for performing the migration using a temporary collection.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    async createTargetConnector(newEntitySchema) {
+        const migrationCollectionName = `${this._config.collection}Migration${Date.now()}`;
+        return new MongoDbEntityStorageConnector({
+            entitySchema: newEntitySchema,
+            config: {
+                ...this._config,
+                collection: migrationCollectionName
+            },
+            partitionContextIds: this._partitionContextIds
+        });
+    }
+    /**
+     * Finalize the migration by dropping the source collection and renaming the migration collection to the original name.
+     * @param targetConnector The connector holding the migrated data in a temporary collection.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The logging component type to use during finalization.
+     * @returns The final connector using the original collection name with the new schema.
+     */
+    async finalizeMigration(targetConnector, options, loggingComponentType) {
+        // With collection-per-partition each partition is a separate collection, so we must
+        // rename every target partition collection to the corresponding source name. We do this
+        // without relying on context so that all partitions are handled in a single call.
+        const targetBase = targetConnector._config.collection;
+        const sourceBase = this._config.collection;
+        const targetDb = targetConnector._client.db(targetConnector._config.database);
+        const sourceDb = this._client.db(this._config.database);
+        // Find all collections the target connector wrote to (exact base name or with a _suffix).
+        const allCollections = await targetDb.listCollections().toArray();
+        const migrationCollections = allCollections.filter(c => c.name === targetBase || c.name.startsWith(`${targetBase}_`));
+        for (const col of migrationCollections) {
+            // Preserve whatever suffix (empty, or "_partitionKey") was appended to the base name.
+            const suffix = col.name.slice(targetBase.length);
+            const finalName = `${sourceBase}${suffix}`;
+            // Drop the existing source collection to free up the name.
+            try {
+                await sourceDb.collection(finalName).drop();
+            }
+            catch { } // collection may not exist yet
+            await targetDb.collection(col.name).rename(finalName);
+        }
+        const finalConnector = new MongoDbEntityStorageConnector({
+            entitySchema: targetConnector._entitySchemaName,
+            config: this._config,
+            partitionContextIds: this._partitionContextIds
+        });
+        if (await finalConnector.bootstrap(loggingComponentType)) {
+            await targetConnector.stop?.();
+            return finalConnector;
+        }
+        throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "finalizeMigrationFailedBootstrap", undefined);
+    }
+    /**
+     * Cleanup a failed or aborted migration by dropping the temporary migration collection.
+     * @param targetConnector The target connector to cleanup.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging.
+     * @returns A promise that resolves when the cleanup is complete.
+     */
+    async cleanupMigration(targetConnector, options, loggingComponentType) {
+        await targetConnector?.teardown?.(loggingComponentType);
+    }
     /**
      * Create a new DB connection configuration.
      * @returns The MongoDb connection configuration.
@@ -316,13 +524,44 @@ export class MongoDbEntityStorageConnector {
         return `mongodb://${host}${portPart}/${database}`;
     }
     /**
-     * Return a Mongo DB collection.
+     * Return a Mongo DB collection for the current partition context.
      * @returns The MongoDb collection.
      * @internal
      */
     async getCollection() {
-        const { database, collection } = this._config;
-        return this._client.db(database).collection(collection);
+        const collectionName = await this.resolveCollectionName(this._config.collection);
+        return this._client.db(this._config.database).collection(collectionName);
+    }
+    /**
+     * Resolve the collection name for a base name, appending the partition key when applicable.
+     * @param base The base collection name.
+     * @returns The resolved collection name.
+     * @internal
+     */
+    async resolveCollectionName(base) {
+        if (!Is.arrayValue(this._partitionContextIds)) {
+            return base;
+        }
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        return Is.stringValue(partitionKey) ? `${base}_${partitionKey.replace(/[\0$]/g, "_")}` : base;
+    }
+    /**
+     * Build a MongoDB filter from optional conditions.
+     * @param conditions The optional entity conditions to include.
+     * @returns The MongoDB filter object.
+     * @internal
+     */
+    buildFilter(conditions) {
+        const filter = {};
+        if (!Is.empty(conditions)) {
+            const finalConditions = {
+                conditions: [conditions],
+                logicalOperator: LogicalOperator.And
+            };
+            this.buildQueryParameters("", finalConditions, filter);
+        }
+        return filter;
     }
     /**
      * Create an MongoDB filter query.
@@ -352,8 +591,17 @@ export class MongoDbEntityStorageConnector {
             }
         }
         else {
-            const prop = objectPath ? `${objectPath}.${condition.property}` : String(condition.property);
-            const comparison = this.mapComparisonOperator(condition.comparison, condition.value);
+            const propertyPath = String(condition.property);
+            const prop = objectPath ? `${objectPath}.${propertyPath}` : propertyPath;
+            const propertyParts = propertyPath.split(".");
+            const schemaLookupName = propertyParts.length > 1 ? propertyParts[0] : propertyPath;
+            const propertySchema = this._entitySchema.properties?.find(p => p.property === schemaLookupName);
+            // For dot-notation paths the leaf field is always a string value; using the root
+            // type directly would send Includes into $elemMatch which does not work for nested
+            // string fields. Keeping String here causes mapComparisonOperator to emit $regex,
+            // which MongoDB handles correctly for both nested object and array traversal.
+            const propertyType = propertyParts.length > 1 ? EntitySchemaPropertyType.String : propertySchema?.type;
+            const comparison = this.mapComparisonOperator(condition.comparison, condition.value, propertyType);
             filter[prop] = comparison;
         }
     }
@@ -361,10 +609,12 @@ export class MongoDbEntityStorageConnector {
      * Map the framework comparison operators to those in MongoDB.
      * @param comparison The comparison operator.
      * @param value The value to compare.
+     * @param type The type of the property from the schema.
      * @returns The MongoDB comparison expression.
+     * @throws GeneralError if the comparison operator is not supported.
      * @internal
      */
-    mapComparisonOperator(comparison, value) {
+    mapComparisonOperator(comparison, value, type) {
         switch (comparison) {
             case ComparisonOperator.Equals:
                 return value;
@@ -381,9 +631,28 @@ export class MongoDbEntityStorageConnector {
             case ComparisonOperator.In:
                 return { $in: Array.isArray(value) ? value : [value] };
             case ComparisonOperator.Includes:
+                // For string fields, use regex for substring matching
+                if (type === EntitySchemaPropertyType.String) {
+                    // Escape special regex characters in the value
+                    const escapedValue = String(value).replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
+                    return { $regex: escapedValue };
+                }
+                // For array and object fields, use $elemMatch
+                if (type === EntitySchemaPropertyType.Array || type === EntitySchemaPropertyType.Object) {
+                    return { $elemMatch: { $eq: value } };
+                }
+                // Fallback to $elemMatch for backwards compatibility
                 return { $elemMatch: { $eq: value } };
             case ComparisonOperator.NotIncludes:
-                return { $elemMatch: { $ne: value } };
+                // For string fields, use negated regex
+                if (type === EntitySchemaPropertyType.String) {
+                    const escapedValue = String(value).replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
+                    return { $not: { $regex: escapedValue } };
+                }
+                // For array/object fields: $ne on an array field matches documents where
+                // none of the array elements equal the value (MongoDB element-wise semantics).
+                // $elemMatch: { $ne: value } is wrong — it matches if *any* element ≠ value.
+                return { $ne: value };
             default:
                 throw new GeneralError(MongoDbEntityStorageConnector.CLASS_NAME, "unsupportedComparisonOperator", { comparison });
         }