@twin.org/entity-storage-connector-dynamodb 0.0.3-next.2 → 0.0.3-next.20

package/dist/es/dynamoDbEntityStorageConnector.js

@@ -1,11 +1,12 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
-import { DynamoDB, QueryCommand, waitUntilTableExists } from "@aws-sdk/client-dynamodb";
-import { DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand } from "@aws-sdk/lib-dynamodb";
+import { BatchWriteItemCommand, DynamoDB, QueryCommand, ScanCommand as RawScanCommand, waitUntilTableExists, waitUntilTableNotExists } from "@aws-sdk/client-dynamodb";
+import { BatchWriteCommand, DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand, ScanCommand } from "@aws-sdk/lib-dynamodb";
 import { unmarshall } from "@aws-sdk/util-dynamodb";
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
-import { BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, Is, ObjectHelper } from "@twin.org/core";
+import { BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, HealthStatus, Is, ObjectHelper } from "@twin.org/core";
 import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, LogicalOperator, SortDirection } from "@twin.org/entity";
+import { EntityStorageHelper } from "@twin.org/entity-storage-models";
 /**
  * Class for performing entity storage operations using Dynamo DB.
  */
@@ -29,6 +30,11 @@ export class DynamoDbEntityStorageConnector {
      * @internal
      */
     static _PARTITION_KEY_VALUE = "root";
+    /**
+     * The name for the schema.
+     * @internal
+     */
+    _entitySchemaName;
     /**
      * The schema for the entity.
      * @internal
@@ -64,8 +70,9 @@ export class DynamoDbEntityStorageConnector {
         }
         Guards.stringValue(DynamoDbEntityStorageConnector.CLASS_NAME, "options.config.region", options.config.region);
         Guards.stringValue(DynamoDbEntityStorageConnector.CLASS_NAME, "options.config.tableName", options.config.tableName);
-        this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._partitionContextIds = options.partitionContextIds;
+        this._entitySchemaName = options.entitySchema;
+        this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
         this._config = options.config;
         this._config.endpoint = Is.stringValue(this._config.endpoint)
@@ -79,6 +86,35 @@ export class DynamoDbEntityStorageConnector {
     className() {
         return DynamoDbEntityStorageConnector.CLASS_NAME;
     }
+    /**
+     * Returns the health status of the component.
+     * @returns The health status of the component.
+     */
+    async health() {
+        try {
+            const dbConnection = this.createConnection();
+            await dbConnection.describeTable({ TableName: this._config.tableName });
+            return [
+                {
+                    source: DynamoDbEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Ok,
+                    description: "healthDescription",
+                    data: { tableName: this._config.tableName }
+                }
+            ];
+        }
+        catch {
+            return [
+                {
+                    source: DynamoDbEntityStorageConnector.CLASS_NAME,
+                    status: HealthStatus.Error,
+                    description: "healthDescription",
+                    message: "connectionFailed",
+                    data: { tableName: this._config.tableName }
+                }
+            ];
+        }
+    }
     /**
      * Get the schema for the entities.
      * @returns The schema for the entities.
@@ -174,7 +210,7 @@ export class DynamoDbEntityStorageConnector {
                 // Wait for table to exist
                 await waitUntilTableExists({
                     client: dbConnection,
-                    maxWaitTime: 60000
+                    maxWaitTime: 60
                 }, {
                     TableName: this._config.tableName
                 });
@@ -250,8 +286,12 @@ export class DynamoDbEntityStorageConnector {
                     }
                 });
                 const response = await docClient.send(getCommand);
-                delete response.Item?.[DynamoDbEntityStorageConnector._PARTITION_KEY];
-                return response.Item;
+                if (response.Item) {
+                    return EntityStorageHelper.unPrepareEntity(response.Item, [
+                        DynamoDbEntityStorageConnector._PARTITION_KEY
+                    ]);
+                }
+                return undefined;
             }
             const finalConditions = {
                 conditions: []
@@ -296,17 +336,21 @@ export class DynamoDbEntityStorageConnector {
         Guards.object(DynamoDbEntityStorageConnector.CLASS_NAME, "entity", entity);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        EntitySchemaHelper.validateEntity(entity, this.getSchema());
-        const id = entity[this._primaryKey.property];
+        const prepared = EntityStorageHelper.prepareEntity(entity, this._entitySchema, partitionKey
+            ? [{ property: DynamoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : [
+                {
+                    property: DynamoDbEntityStorageConnector._PARTITION_KEY,
+                    value: DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
+                }
+            ], { nullBehavior: "omit" });
+        const id = prepared[this._primaryKey.property];
         try {
             const docClient = this.createDocClient();
             const { conditionExpression, attributeNames, attributeValues } = this.buildConditionExpression(conditions);
             const putCommand = new PutCommand({
                 TableName: this._config.tableName,
-                Item: {
-                    [DynamoDbEntityStorageConnector._PARTITION_KEY]: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE,
-                    ...entity
-                },
+                Item: prepared,
                 // Only set the condition expression if we have conditions to match
                 // and the primary key exists, otherwise we are creating a new object
                 ConditionExpression: Is.stringValue(conditionExpression)
@@ -331,6 +375,93 @@ export class DynamoDbEntityStorageConnector {
             }, err);
         }
     }
+    /**
+     * Set multiple entities in a batch.
+     * @param entities The entities to set.
+     * @returns Nothing.
+     */
+    async setBatch(entities) {
+        Guards.arrayValue(DynamoDbEntityStorageConnector.CLASS_NAME, "entities", entities);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        const preparedEntities = entities.map(entity => EntityStorageHelper.prepareEntity(entity, this._entitySchema, partitionKey
+            ? [{ property: DynamoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : [
+                {
+                    property: DynamoDbEntityStorageConnector._PARTITION_KEY,
+                    value: DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
+                }
+            ], { nullBehavior: "omit" }));
+        try {
+            const docClient = this.createDocClient();
+            const chunkSize = 25;
+            for (let i = 0; i < preparedEntities.length; i += chunkSize) {
+                const chunk = preparedEntities.slice(i, i + chunkSize);
+                await docClient.send(new BatchWriteCommand({
+                    RequestItems: {
+                        [this._config.tableName]: chunk.map(entity => ({
+                            PutRequest: {
+                                Item: entity
+                            }
+                        }))
+                    }
+                }));
+            }
+        }
+        catch (err) {
+            if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
+                throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "tableDoesNotExist", { tableName: this._config.tableName }, err);
+            }
+            throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "setBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Empty the entity storage.
+     * @returns Nothing.
+     */
+    async empty() {
+        try {
+            const contextIds = await ContextIdStore.getContextIds();
+            const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+            const pKey = partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE;
+            const docClient = this.createDocClient();
+            const chunkSize = 25;
+            let exclusiveStartKey;
+            do {
+                const scanResult = await docClient.send(new ScanCommand({
+                    TableName: this._config.tableName,
+                    FilterExpression: "#partitionId = :partitionId",
+                    ExpressionAttributeNames: {
+                        "#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
+                    },
+                    ExpressionAttributeValues: {
+                        ":partitionId": pKey
+                    },
+                    ExclusiveStartKey: exclusiveStartKey
+                }));
+                const items = scanResult.Items ?? [];
+                for (let i = 0; i < items.length; i += chunkSize) {
+                    const chunk = items.slice(i, i + chunkSize);
+                    await docClient.send(new BatchWriteCommand({
+                        RequestItems: {
+                            [this._config.tableName]: chunk.map((item) => ({
+                                DeleteRequest: {
+                                    Key: {
+                                        [DynamoDbEntityStorageConnector._PARTITION_KEY]: item[DynamoDbEntityStorageConnector._PARTITION_KEY],
+                                        [this._primaryKey.property]: item[this._primaryKey.property]
+                                    }
+                                }
+                            }))
+                        }
+                    }));
+                }
+                exclusiveStartKey = scanResult.LastEvaluatedKey;
+            } while (exclusiveStartKey);
+        }
+        catch (err) {
+            throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "emptyFailed", undefined, err);
+        }
+    }
     /**
      * Remove the entity.
      * @param id The id of the entity to remove.
@@ -370,6 +501,77 @@ export class DynamoDbEntityStorageConnector {
             }, err);
         }
     }
+    /**
+     * Remove multiple entities by their IDs in a batch.
+     * @param ids The ids of the entities to remove.
+     * @returns Nothing.
+     */
+    async removeBatch(ids) {
+        Guards.arrayValue(DynamoDbEntityStorageConnector.CLASS_NAME, "ids", ids);
+        const contextIds = await ContextIdStore.getContextIds();
+        const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+        try {
+            const docClient = this.createDocClient();
+            const chunkSize = 25;
+            const primaryKeyProperty = this._primaryKey.property;
+            for (let i = 0; i < ids.length; i += chunkSize) {
+                const chunk = ids.slice(i, i + chunkSize);
+                await docClient.send(new BatchWriteCommand({
+                    RequestItems: {
+                        [this._config.tableName]: chunk.map(id => ({
+                            DeleteRequest: {
+                                Key: {
+                                    [primaryKeyProperty]: id,
+                                    [DynamoDbEntityStorageConnector._PARTITION_KEY]: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
+                                }
+                            }
+                        }))
+                    }
+                }));
+            }
+        }
+        catch (err) {
+            throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "removeBatchFailed", undefined, err);
+        }
+    }
+    /**
+     * Teardown the entity storage by deleting the underlying table.
+     * @param nodeLoggingComponentType The node logging component type.
+     * @returns True if the teardown process was successful.
+     */
+    async teardown(nodeLoggingComponentType) {
+        const nodeLogging = ComponentFactory.getIfExists(nodeLoggingComponentType);
+        await nodeLogging?.log({
+            level: "info",
+            source: DynamoDbEntityStorageConnector.CLASS_NAME,
+            ts: Date.now(),
+            message: "tableDeleting",
+            data: { tableName: this._config.tableName }
+        });
+        try {
+            const dbConnection = this.createConnection();
+            await dbConnection.deleteTable({ TableName: this._config.tableName });
+            await waitUntilTableNotExists({ client: dbConnection, maxWaitTime: 60 }, { TableName: this._config.tableName });
+            await nodeLogging?.log({
+                level: "info",
+                source: DynamoDbEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "tableDeleted",
+                data: { tableName: this._config.tableName }
+            });
+            return true;
+        }
+        catch (err) {
+            await nodeLogging?.log({
+                level: "error",
+                source: DynamoDbEntityStorageConnector.CLASS_NAME,
+                ts: Date.now(),
+                message: "teardownFailed",
+                error: BaseError.fromError(err)
+            });
+            return false;
+        }
+    }
     /**
      * Find all the entities which match the conditions.
      * @param conditions The conditions to match for the entities.
@@ -386,15 +588,191 @@ export class DynamoDbEntityStorageConnector {
         return this.internalQuery(conditions, sortProperties, properties, cursor, limit, undefined, partitionKey);
     }
     /**
-     * Delete the table.
-     * @returns Nothing.
+     * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
+     * @returns The total count of entities in the storage.
      */
-    async tableDelete() {
+    async count(conditions) {
         try {
+            const contextIds = await ContextIdStore.getContextIds();
+            const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+            const attributeNames = {
+                "#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
+            };
+            const attributeValues = {
+                ":partitionId": {
+                    S: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
+                }
+            };
+            const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues);
             const dbConnection = this.createConnection();
-            await dbConnection.deleteTable({ TableName: this._config.tableName });
+            let total = 0;
+            let exclusiveStartKey;
+            do {
+                const result = await dbConnection.send(new QueryCommand({
+                    TableName: this._config.tableName,
+                    Select: "COUNT",
+                    KeyConditionExpression: "#partitionId = :partitionId",
+                    FilterExpression: Is.stringValue(expressions.filterCondition)
+                        ? expressions.filterCondition
+                        : undefined,
+                    ExpressionAttributeNames: attributeNames,
+                    ExpressionAttributeValues: attributeValues,
+                    ExclusiveStartKey: exclusiveStartKey
+                }));
+                total += result.Count ?? 0;
+                exclusiveStartKey = result.LastEvaluatedKey;
+            } while (exclusiveStartKey);
+            return total;
+        }
+        catch (err) {
+            throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "countFailed", undefined, err);
+        }
+    }
+    /**
+     * Get a unique list of all the context ids from the storage.
+     * @returns The list of unique context ids.
+     */
+    async getPartitionContextIds() {
+        if (!Is.arrayValue(this._partitionContextIds)) {
+            return [];
+        }
+        const contextIdsMap = {};
+        try {
+            const docClient = this.createDocClient();
+            let exclusiveStartKey;
+            do {
+                const scanResult = await docClient.send(new ScanCommand({
+                    TableName: this._config.tableName,
+                    ProjectionExpression: "#partitionId",
+                    ExpressionAttributeNames: {
+                        "#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
+                    },
+                    ExclusiveStartKey: exclusiveStartKey
+                }));
+                for (const item of scanResult.Items ?? []) {
+                    const partitionId = item[DynamoDbEntityStorageConnector._PARTITION_KEY];
+                    if (Is.stringValue(partitionId) && !(partitionId in contextIdsMap)) {
+                        contextIdsMap[partitionId] = ContextIdHelper.shortSplit(this._partitionContextIds ?? [], partitionId);
+                    }
+                }
+                exclusiveStartKey = scanResult.LastEvaluatedKey;
+            } while (exclusiveStartKey);
+        }
+        catch (err) {
+            throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "getPartitionContextIdsFailed", undefined, err);
+        }
+        return Object.values(contextIdsMap);
+    }
+    /**
+     * Create the target connector for performing the migration it will use a temporary storage location.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    async createTargetConnector(newEntitySchema) {
+        // We create a new table for the migration with a unique name to avoid conflicts with the existing table
+        // This table will be swapped with the existing table once the migration is finalized.
+        const migrationTableName = `${this._config.tableName}Migration${Date.now()}`;
+        return new DynamoDbEntityStorageConnector({
+            entitySchema: newEntitySchema,
+            config: {
+                ...this._config,
+                tableName: migrationTableName
+            },
+            partitionContextIds: this._partitionContextIds
+        });
+    }
+    /**
+     * Finalize the migration by tearing down the old connector and replacing it with the new one.
+     * @param targetConnector The target connector to finalize the migration with.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The logging component type to use for logging during the migration finalization.
+     * @returns A promise that resolves when the migration is finalized.
+     */
+    async finalizeMigration(targetConnector, options, loggingComponentType) {
+        // There is no rename operation in DynamoDB so we have to create a new table with the original name and copy the data over
+        // Teardown the existing table with the original name to free up the name for the new table
+        await this.teardown(loggingComponentType);
+        // Create a new connector with the original table name but with the new schema
+        // and copy the data from the migration table to the new table using batch operations
+        const finalConnector = new DynamoDbEntityStorageConnector({
+            entitySchema: targetConnector._entitySchemaName,
+            config: this._config,
+            partitionContextIds: this._partitionContextIds
+        });
+        if (await finalConnector.bootstrap(loggingComponentType)) {
+            // Since there is no rename, we need to copy the data from the migration table to the new table
+            const partitions = await targetConnector.getPartitionContextIds();
+            const batchSize = options?.batchSize ?? DynamoDbEntityStorageConnector._DEFAULT_LIMIT;
+            await this.bulkCopy(targetConnector, finalConnector, partitions, batchSize);
+            await targetConnector.teardown(loggingComponentType);
+            return finalConnector;
+        }
+        throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "finalizeMigrationFailedBootstrap", undefined);
+    }
+    /**
+     * Cleanup the migration if a migration fails or needs to be aborted.
+     * @param targetConnector The target connector to cleanup the migration with.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is cleaned up.
+     */
+    async cleanupMigration(targetConnector, options, loggingComponentType) {
+        // If something failed the only thing to cleanup is the migration table
+        await targetConnector?.teardown?.(loggingComponentType);
+    }
+    /**
+     * Copy all entities from sourceConnector to destConnector, paging through each partition.
+     * @param sourceConnector The connector to read entities from.
+     * @param destConnector The connector to write entities to.
+     * @param partitions The partition list returned by getPartitionContextIds.
+     * @param batchSize The number of entities to read per page.
+     * @internal
+     */
+    async bulkCopy(sourceConnector, destConnector, partitions, batchSize) {
+        let partitionList;
+        if (Is.arrayValue(partitions)) {
+            partitionList = partitions;
+        }
+        else if (Is.arrayValue(sourceConnector._partitionContextIds)) {
+            partitionList = [];
+        }
+        else {
+            partitionList = [{}];
+        }
+        const dbConnection = sourceConnector.createConnection();
+        const chunkSize = 25;
+        for (let i = 0; i < partitionList.length; i++) {
+            const partitionKey = ContextIdHelper.combinedContextKey(partitionList[i], sourceConnector._partitionContextIds) ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE;
+            let exclusiveStartKey;
+            do {
+                const { Items: items, LastEvaluatedKey: lastKey } = await dbConnection.send(new QueryCommand({
+                    TableName: sourceConnector._config.tableName,
+                    KeyConditionExpression: `#${DynamoDbEntityStorageConnector._PARTITION_KEY} = :${DynamoDbEntityStorageConnector._PARTITION_KEY}`,
+                    ExpressionAttributeNames: {
+                        [`#${DynamoDbEntityStorageConnector._PARTITION_KEY}`]: DynamoDbEntityStorageConnector._PARTITION_KEY
+                    },
+                    ExpressionAttributeValues: {
+                        [`:${DynamoDbEntityStorageConnector._PARTITION_KEY}`]: { S: partitionKey }
+                    },
+                    Limit: batchSize,
+                    ExclusiveStartKey: exclusiveStartKey
+                }));
+                exclusiveStartKey = lastKey;
+                if (Is.arrayValue(items)) {
+                    for (let j = 0; j < items.length; j += chunkSize) {
+                        const chunk = items.slice(j, j + chunkSize);
+                        await dbConnection.send(new BatchWriteItemCommand({
+                            RequestItems: {
+                                [destConnector._config.tableName]: chunk.map(item => ({
+                                    PutRequest: { Item: item }
+                                }))
+                            }
+                        }));
+                    }
+                }
+            } while (exclusiveStartKey);
         }
-        catch { }
     }
     /**
      * Create the parameters for a query.
@@ -410,19 +788,44 @@ export class DynamoDbEntityStorageConnector {
         if (Is.undefined(condition)) {
             return {
                 keyCondition: "",
-                filterCondition: ""
+                filterCondition: "",
+                requiresScan: false
             };
         }
         if ("conditions" in condition) {
             if (condition.conditions.length === 0) {
                 return {
                     keyCondition: "",
-                    filterCondition: ""
+                    filterCondition: "",
+                    requiresScan: false
                 };
             }
             // It's a group of comparisons, so check the individual items and combine with the logical operator
             const joinConditions = condition.conditions.map(c => this.buildQueryParameters(objectPath, c, attributeNames, attributeValues, secondaryIndex));
             const logicalOperator = this.mapConditionalOperator(condition.logicalOperator);
+            // DynamoDB does not support OR in KeyConditionExpression, so when the operator
+            // is OR we must move all conditions (including key conditions) into FilterExpression.
+            if (condition.logicalOperator === LogicalOperator.Or) {
+                const parts = joinConditions
+                    .map(j => {
+                    const subParts = [j.keyCondition.trim(), j.filterCondition.trim()].filter(s => s.length > 0);
+                    if (subParts.length === 0) {
+                        return "";
+                    }
+                    if (subParts.length === 1) {
+                        return subParts[0];
+                    }
+                    return `(${subParts.join(" AND ")})`;
+                })
+                    .filter(s => s.length > 0);
+                const hasKeyConditions = joinConditions.some(j => j.keyCondition.length > 0);
+                const filterCondition = parts.join(" OR ");
+                return {
+                    keyCondition: "",
+                    filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : "",
+                    requiresScan: hasKeyConditions
+                };
+            }
             const keyCondition = joinConditions
                 .filter(j => j.keyCondition.length > 0)
                 .map(j => j.keyCondition)
@@ -433,7 +836,8 @@ export class DynamoDbEntityStorageConnector {
                 .join(` ${logicalOperator} `);
             return {
                 keyCondition: Is.stringValue(keyCondition) ? ` (${keyCondition}) ` : "",
-                filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : ""
+                filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : "",
+                requiresScan: joinConditions.some(j => j.requiresScan)
             };
         }
         const schemaProp = this._entitySchema.properties?.find(p => p.property === condition.property);
@@ -442,7 +846,8 @@ export class DynamoDbEntityStorageConnector {
         const isKey = schemaProp?.isPrimary ?? (schemaProp?.isSecondary && schemaProp?.property === secondaryIndex);
         return {
             keyCondition: isKey ? comparison : "",
-            filterCondition: !isKey ? comparison : ""
+            filterCondition: !isKey ? comparison : "",
+            requiresScan: false
         };
     }
     /**
@@ -464,6 +869,8 @@ export class DynamoDbEntityStorageConnector {
         prop += comparator.property;
         let attributeName = this.populateAttributeNames(prop, attributeNames);
         if (Is.empty(comparator.value)) {
+            // With "omit" storage, optional null/undefined fields are absent from the item entirely.
+            // attribute_not_exists matches absent attributes; attribute_exists matches present ones.
             if (comparator.comparison === ComparisonOperator.Equals) {
                 return `attribute_not_exists(${attributeName})`;
             }
@@ -471,7 +878,13 @@ export class DynamoDbEntityStorageConnector {
                 return `attribute_exists(${attributeName})`;
             }
         }
-        let propName = `:${attributeName.replace(/\./g, "").replace(/#/g, "")}`;
+        const basePropName = `:${attributeName.replace(/\./g, "").replace(/#/g, "")}`;
+        let propName = basePropName;
+        let propSuffix = 0;
+        while (!Is.undefined(attributeValues[propName])) {
+            propSuffix++;
+            propName = `${basePropName}${propSuffix}`;
+        }
         if (Is.array(comparator.value)) {
             const dbValues = comparator.value.map(v => this.propertyToDbValue(v, type));
             const arrAttributeNames = [];
@@ -508,7 +921,7 @@ export class DynamoDbEntityStorageConnector {
             return `contains(${attributeName}, ${propName})`;
         }
         else if (comparator.comparison === ComparisonOperator.NotIncludes) {
-            return `notContains(${attributeName}, ${propName})`;
+            return `NOT contains(${attributeName}, ${propName})`;
         }
         else if (comparator.comparison === ComparisonOperator.In) {
             return `${propName} IN ${attributeName}`;
@@ -608,6 +1021,9 @@ export class DynamoDbEntityStorageConnector {
      * @internal
      */
     createConnectionConfig() {
+        const requestHandler = Is.number(this._config.connectionTimeoutMs)
+            ? { requestTimeout: this._config.connectionTimeoutMs }
+            : undefined;
         if (Is.stringValue(this._config.secretAccessKey) &&
             Is.stringValue(this._config.accessKeyId) &&
             this._config.authMode === "credentials") {
@@ -617,12 +1033,16 @@ export class DynamoDbEntityStorageConnector {
                     secretAccessKey: this._config.secretAccessKey
                 },
                 endpoint: this._config.endpoint,
-                region: this._config.region
+                region: this._config.region,
+                requestHandler,
+                maxAttempts: this._config.maxAttempts
             };
         }
         return {
             endpoint: this._config.endpoint,
-            region: this._config.region
+            region: this._config.region,
+            requestHandler,
+            maxAttempts: this._config.maxAttempts
         };
     }
     /**
@@ -634,8 +1054,9 @@ export class DynamoDbEntityStorageConnector {
     async tableExists(tableName) {
         try {
             const dbConnection = this.createConnection();
-            await dbConnection.describeTable({ TableName: tableName });
-            return true;
+            const result = await dbConnection.describeTable({ TableName: tableName });
+            // A table in DELETING state should not be treated as existing
+            return result.Table?.TableStatus !== "DELETING";
         }
         catch {
             return false;
@@ -690,6 +1111,49 @@ export class DynamoDbEntityStorageConnector {
                 }
             };
             const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues, secondaryIndex);
+            // OR conditions on primary key attributes can't use KeyConditionExpression or
+            // FilterExpression in a QueryCommand — fall back to a full table ScanCommand.
+            if (expressions.requiresScan) {
+                let scanFilter = "#partitionId = :partitionId";
+                if (Is.stringValue(expressions.filterCondition)) {
+                    scanFilter += ` AND ${expressions.filterCondition.trim()}`;
+                }
+                const dbConnection = this.createConnection();
+                const matchingItems = [];
+                let scanStartKey = Is.empty(cursor)
+                    ? undefined
+                    : ObjectHelper.fromBytes(Converter.base64ToBytes(cursor));
+                do {
+                    const scanResult = await dbConnection.send(new RawScanCommand({
+                        TableName: this._config.tableName,
+                        FilterExpression: scanFilter,
+                        ExpressionAttributeNames: attributeNames,
+                        ExpressionAttributeValues: attributeValues,
+                        ProjectionExpression: properties?.map(p => p).join(", "),
+                        ExclusiveStartKey: scanStartKey
+                    }));
+                    matchingItems.push(...(scanResult.Items ?? []));
+                    scanStartKey = scanResult.LastEvaluatedKey;
+                } while (!Is.empty(scanStartKey));
+                const hasMore = matchingItems.length > returnSize;
+                const returnedRawItems = hasMore ? matchingItems.slice(0, returnSize) : matchingItems;
+                let resultCursor;
+                if (hasMore) {
+                    const lastRawItem = returnedRawItems[returnedRawItems.length - 1];
+                    const syntheticKey = {
+                        [DynamoDbEntityStorageConnector._PARTITION_KEY]: lastRawItem[DynamoDbEntityStorageConnector._PARTITION_KEY],
+                        [this._primaryKey.property]: lastRawItem[this._primaryKey.property]
+                    };
+                    resultCursor = Converter.bytesToBase64(ObjectHelper.toBytes(syntheticKey));
+                }
+                const scanEntities = returnedRawItems.map(item => {
+                    const unmarshalled = unmarshall(item);
+                    return EntityStorageHelper.unPrepareEntity(unmarshalled, [
+                        DynamoDbEntityStorageConnector._PARTITION_KEY
+                    ]);
+                });
+                return { entities: scanEntities, cursor: resultCursor };
+            }
             let keyExpression = "#partitionId = :partitionId";
             if (expressions.keyCondition.length > 0) {
                 keyExpression += ` AND ${expressions.keyCondition}`;
@@ -704,7 +1168,7 @@ export class DynamoDbEntityStorageConnector {
                 ExpressionAttributeNames: attributeNames,
                 ExpressionAttributeValues: attributeValues,
                 ProjectionExpression: properties?.map(p => p).join(", "),
-                Limit: returnSize,
+                Limit: returnSize + 1,
                 ScanIndexForward: scanAscending,
                 ExclusiveStartKey: Is.empty(cursor)
                     ? undefined
@@ -712,20 +1176,28 @@ export class DynamoDbEntityStorageConnector {
             });
             const connection = this.createDocClient();
             const results = await connection.send(query);
-            let entities = [];
-            if (Is.arrayValue(results.Items)) {
-                entities = results.Items.map(item => {
-                    const unmarshalled = unmarshall(item);
-                    delete unmarshalled[DynamoDbEntityStorageConnector._PARTITION_KEY];
-                    return unmarshalled;
-                });
+            const rawItems = results.Items ?? [];
+            const hasMore = rawItems.length > returnSize;
+            const returnedRawItems = hasMore ? rawItems.slice(0, returnSize) : rawItems;
+            let resultCursor;
+            if (hasMore) {
+                const lastRawItem = returnedRawItems[returnedRawItems.length - 1];
+                const syntheticKey = {
+                    [DynamoDbEntityStorageConnector._PARTITION_KEY]: lastRawItem[DynamoDbEntityStorageConnector._PARTITION_KEY],
+                    [this._primaryKey.property]: lastRawItem[this._primaryKey.property]
+                };
+                if (Is.stringValue(secondaryIndex)) {
+                    syntheticKey[secondaryIndex] = lastRawItem[secondaryIndex];
+                }
+                resultCursor = Converter.bytesToBase64(ObjectHelper.toBytes(syntheticKey));
             }
-            return {
-                entities,
-                cursor: Is.empty(results.LastEvaluatedKey)
-                    ? undefined
-                    : Converter.bytesToBase64(ObjectHelper.toBytes(results.LastEvaluatedKey))
-            };
+            const entities = returnedRawItems.map(item => {
+                const unmarshalled = unmarshall(item);
+                return EntityStorageHelper.unPrepareEntity(unmarshalled, [
+                    DynamoDbEntityStorageConnector._PARTITION_KEY
+                ]);
+            });
+            return { entities, cursor: resultCursor };
         }
         catch (err) {
             if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {