@twin.org/entity-storage-connector-dynamodb 0.0.3-next.13 → 0.0.3-next.14

package/README.md

@@ -13,8 +13,7 @@ npm install @twin.org/entity-storage-connector-dynamodb
 To perform testing of this component it may be necessary to launch a local instance to communicate with.
 
 ```shell
-docker pull amazon/dynamodb-local:latest
-docker run -d --name twin-entity-storage-dynamodb -p 10000:8000 amazon/dynamodb-local:latest
+docker run -d --name twin-entity-storage-dynamodb -p 8000:8000 amazon/dynamodb-local
 ```
 
 ## Examples

package/dist/es/dynamoDbEntityStorageConnector.js

@@ -1,11 +1,12 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
-import { DynamoDB, QueryCommand, waitUntilTableExists } from "@aws-sdk/client-dynamodb";
+import { BatchWriteItemCommand, DynamoDB, QueryCommand, ScanCommand as RawScanCommand, waitUntilTableExists, waitUntilTableNotExists } from "@aws-sdk/client-dynamodb";
 import { BatchWriteCommand, DeleteCommand, DynamoDBDocumentClient, GetCommand, PutCommand, ScanCommand } from "@aws-sdk/lib-dynamodb";
 import { unmarshall } from "@aws-sdk/util-dynamodb";
 import { ContextIdHelper, ContextIdStore } from "@twin.org/context";
 import { BaseError, Coerce, ComponentFactory, Converter, GeneralError, Guards, HealthStatus, Is, ObjectHelper } from "@twin.org/core";
 import { ComparisonOperator, EntitySchemaFactory, EntitySchemaHelper, LogicalOperator, SortDirection } from "@twin.org/entity";
+import { EntityHelper } from "@twin.org/entity-storage-models";
 /**
  * Class for performing entity storage operations using Dynamo DB.
  */
@@ -29,6 +30,11 @@ export class DynamoDbEntityStorageConnector {
      * @internal
      */
     static _PARTITION_KEY_VALUE = "root";
+    /**
+     * The name for the schema.
+     * @internal
+     */
+    _entitySchemaName;
     /**
      * The schema for the entity.
      * @internal
@@ -64,8 +70,9 @@ export class DynamoDbEntityStorageConnector {
         }
         Guards.stringValue(DynamoDbEntityStorageConnector.CLASS_NAME, "options.config.region", options.config.region);
         Guards.stringValue(DynamoDbEntityStorageConnector.CLASS_NAME, "options.config.tableName", options.config.tableName);
-        this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._partitionContextIds = options.partitionContextIds;
+        this._entitySchemaName = options.entitySchema;
+        this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
         this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
         this._config = options.config;
         this._config.endpoint = Is.stringValue(this._config.endpoint)
@@ -203,7 +210,7 @@ export class DynamoDbEntityStorageConnector {
                 // Wait for table to exist
                 await waitUntilTableExists({
                     client: dbConnection,
-                    maxWaitTime: 60000
+                    maxWaitTime: 60
                 }, {
                     TableName: this._config.tableName
                 });
@@ -279,8 +286,12 @@ export class DynamoDbEntityStorageConnector {
                     }
                 });
                 const response = await docClient.send(getCommand);
-                delete response.Item?.[DynamoDbEntityStorageConnector._PARTITION_KEY];
-                return response.Item;
+                if (response.Item) {
+                    return EntityHelper.unPrepareEntity(response.Item, [
+                        DynamoDbEntityStorageConnector._PARTITION_KEY
+                    ]);
+                }
+                return undefined;
             }
             const finalConditions = {
                 conditions: []
@@ -325,17 +336,21 @@ export class DynamoDbEntityStorageConnector {
         Guards.object(DynamoDbEntityStorageConnector.CLASS_NAME, "entity", entity);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        EntitySchemaHelper.validateEntity(entity, this.getSchema());
-        const id = entity[this._primaryKey.property];
+        const prepared = EntityHelper.prepareEntity(entity, this._entitySchema, partitionKey
+            ? [{ property: DynamoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : [
+                {
+                    property: DynamoDbEntityStorageConnector._PARTITION_KEY,
+                    value: DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
+                }
+            ]);
+        const id = prepared[this._primaryKey.property];
         try {
             const docClient = this.createDocClient();
             const { conditionExpression, attributeNames, attributeValues } = this.buildConditionExpression(conditions);
             const putCommand = new PutCommand({
                 TableName: this._config.tableName,
-                Item: {
-                    [DynamoDbEntityStorageConnector._PARTITION_KEY]: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE,
-                    ...entity
-                },
+                Item: prepared,
                 // Only set the condition expression if we have conditions to match
                 // and the primary key exists, otherwise we are creating a new object
                 ConditionExpression: Is.stringValue(conditionExpression)
@@ -369,22 +384,24 @@ export class DynamoDbEntityStorageConnector {
         Guards.arrayValue(DynamoDbEntityStorageConnector.CLASS_NAME, "entities", entities);
         const contextIds = await ContextIdStore.getContextIds();
         const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
-        for (const entity of entities) {
-            EntitySchemaHelper.validateEntity(entity, this.getSchema());
-        }
+        const preparedEntities = entities.map(entity => EntityHelper.prepareEntity(entity, this._entitySchema, partitionKey
+            ? [{ property: DynamoDbEntityStorageConnector._PARTITION_KEY, value: partitionKey }]
+            : [
+                {
+                    property: DynamoDbEntityStorageConnector._PARTITION_KEY,
+                    value: DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
+                }
+            ]));
         try {
             const docClient = this.createDocClient();
             const chunkSize = 25;
-            for (let i = 0; i < entities.length; i += chunkSize) {
-                const chunk = entities.slice(i, i + chunkSize);
+            for (let i = 0; i < preparedEntities.length; i += chunkSize) {
+                const chunk = preparedEntities.slice(i, i + chunkSize);
                 await docClient.send(new BatchWriteCommand({
                     RequestItems: {
                         [this._config.tableName]: chunk.map(entity => ({
                             PutRequest: {
-                                Item: {
-                                    [DynamoDbEntityStorageConnector._PARTITION_KEY]: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE,
-                                    ...entity
-                                }
+                                Item: entity
                             }
                         }))
                     }
@@ -534,6 +551,7 @@ export class DynamoDbEntityStorageConnector {
         try {
             const dbConnection = this.createConnection();
             await dbConnection.deleteTable({ TableName: this._config.tableName });
+            await waitUntilTableNotExists({ client: dbConnection, maxWaitTime: 60 }, { TableName: this._config.tableName });
             await nodeLogging?.log({
                 level: "info",
                 source: DynamoDbEntityStorageConnector.CLASS_NAME,
@@ -571,12 +589,22 @@ export class DynamoDbEntityStorageConnector {
     }
     /**
      * Count all the entities which match the conditions.
+     * @param conditions The optional conditions to match for the entities.
      * @returns The total count of entities in the storage.
      */
-    async count() {
+    async count(conditions) {
         try {
             const contextIds = await ContextIdStore.getContextIds();
             const partitionKey = ContextIdHelper.combinedContextKey(contextIds, this._partitionContextIds);
+            const attributeNames = {
+                "#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
+            };
+            const attributeValues = {
+                ":partitionId": {
+                    S: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
+                }
+            };
+            const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues);
             const dbConnection = this.createConnection();
             let total = 0;
             let exclusiveStartKey;
@@ -585,14 +613,11 @@ export class DynamoDbEntityStorageConnector {
                     TableName: this._config.tableName,
                     Select: "COUNT",
                     KeyConditionExpression: "#partitionId = :partitionId",
-                    ExpressionAttributeNames: {
-                        "#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
-                    },
-                    ExpressionAttributeValues: {
-                        ":partitionId": {
-                            S: partitionKey ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE
-                        }
-                    },
+                    FilterExpression: Is.stringValue(expressions.filterCondition)
+                        ? expressions.filterCondition
+                        : undefined,
+                    ExpressionAttributeNames: attributeNames,
+                    ExpressionAttributeValues: attributeValues,
                     ExclusiveStartKey: exclusiveStartKey
                 }));
                 total += result.Count ?? 0;
@@ -604,6 +629,151 @@ export class DynamoDbEntityStorageConnector {
             throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "countFailed", undefined, err);
         }
     }
+    /**
+     * Get a unique list of all the context ids from the storage.
+     * @returns The list of unique context ids.
+     */
+    async getPartitionContextIds() {
+        if (!Is.arrayValue(this._partitionContextIds)) {
+            return [];
+        }
+        const contextIdsMap = {};
+        try {
+            const docClient = this.createDocClient();
+            let exclusiveStartKey;
+            do {
+                const scanResult = await docClient.send(new ScanCommand({
+                    TableName: this._config.tableName,
+                    ProjectionExpression: "#partitionId",
+                    ExpressionAttributeNames: {
+                        "#partitionId": DynamoDbEntityStorageConnector._PARTITION_KEY
+                    },
+                    ExclusiveStartKey: exclusiveStartKey
+                }));
+                for (const item of scanResult.Items ?? []) {
+                    const partitionId = item[DynamoDbEntityStorageConnector._PARTITION_KEY];
+                    if (Is.stringValue(partitionId) && !(partitionId in contextIdsMap)) {
+                        contextIdsMap[partitionId] = ContextIdHelper.shortSplit(this._partitionContextIds ?? [], partitionId);
+                    }
+                }
+                exclusiveStartKey = scanResult.LastEvaluatedKey;
+            } while (exclusiveStartKey);
+        }
+        catch (err) {
+            throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "getPartitionContextIdsFailed", undefined, err);
+        }
+        return Object.values(contextIdsMap);
+    }
+    /**
+     * Create the target connector for performing the migration it will use a temporary storage location.
+     * @param newEntitySchema The name of the new entity schema to create the connector for.
+     * @returns Connector for performing the migration.
+     */
+    async createTargetConnector(newEntitySchema) {
+        // We create a new table for the migration with a unique name to avoid conflicts with the existing table
+        // This table will be swapped with the existing table once the migration is finalized.
+        const migrationTableName = `${this._config.tableName}Migration${Date.now()}`;
+        return new DynamoDbEntityStorageConnector({
+            entitySchema: newEntitySchema,
+            config: {
+                ...this._config,
+                tableName: migrationTableName
+            },
+            partitionContextIds: this._partitionContextIds
+        });
+    }
+    /**
+     * Finalize the migration by tearing down the old connector and replacing it with the new one.
+     * @param targetConnector The target connector to finalize the migration with.
+     * @param options The options to control how the migration is finalized.
+     * @param loggingComponentType The logging component type to use for logging during the migration finalization.
+     * @returns A promise that resolves when the migration is finalized.
+     */
+    async finalizeMigration(targetConnector, options, loggingComponentType) {
+        // There is no rename operation in DynamoDB so we have to create a new table with the original name and copy the data over
+        // Teardown the existing table with the original name to free up the name for the new table
+        await this.teardown(loggingComponentType);
+        // Create a new connector with the original table name but with the new schema
+        // and copy the data from the migration table to the new table using batch operations
+        const finalConnector = new DynamoDbEntityStorageConnector({
+            entitySchema: targetConnector._entitySchemaName,
+            config: this._config,
+            partitionContextIds: this._partitionContextIds
+        });
+        if (await finalConnector.bootstrap(loggingComponentType)) {
+            // Since there is no rename, we need to copy the data from the migration table to the new table
+            const partitions = await targetConnector.getPartitionContextIds();
+            const batchSize = options?.batchSize ?? DynamoDbEntityStorageConnector._DEFAULT_LIMIT;
+            await this.bulkCopy(targetConnector, finalConnector, partitions, batchSize);
+            await targetConnector.teardown(loggingComponentType);
+            return finalConnector;
+        }
+        throw new GeneralError(DynamoDbEntityStorageConnector.CLASS_NAME, "finalizeMigrationFailedBootstrap", undefined);
+    }
+    /**
+     * Cleanup the migration if a migration fails or needs to be aborted.
+     * @param targetConnector The target connector to cleanup the migration with.
+     * @param options The options to control how the migration is cleaned up.
+     * @param loggingComponentType The optional component type to use for logging the migration progress.
+     * @returns A promise that resolves when the migration is cleaned up.
+     */
+    async cleanupMigration(targetConnector, options, loggingComponentType) {
+        // If something failed the only thing to cleanup is the migration table
+        await targetConnector?.teardown?.(loggingComponentType);
+    }
+    /**
+     * Copy all entities from sourceConnector to destConnector, paging through each partition.
+     * @param sourceConnector The connector to read entities from.
+     * @param destConnector The connector to write entities to.
+     * @param partitions The partition list returned by getPartitionContextIds.
+     * @param batchSize The number of entities to read per page.
+     * @internal
+     */
+    async bulkCopy(sourceConnector, destConnector, partitions, batchSize) {
+        let partitionList;
+        if (Is.arrayValue(partitions)) {
+            partitionList = partitions;
+        }
+        else if (Is.arrayValue(sourceConnector._partitionContextIds)) {
+            partitionList = [];
+        }
+        else {
+            partitionList = [{}];
+        }
+        const dbConnection = sourceConnector.createConnection();
+        const chunkSize = 25;
+        for (let i = 0; i < partitionList.length; i++) {
+            const partitionKey = ContextIdHelper.combinedContextKey(partitionList[i], sourceConnector._partitionContextIds) ?? DynamoDbEntityStorageConnector._PARTITION_KEY_VALUE;
+            let exclusiveStartKey;
+            do {
+                const { Items: items, LastEvaluatedKey: lastKey } = await dbConnection.send(new QueryCommand({
+                    TableName: sourceConnector._config.tableName,
+                    KeyConditionExpression: `#${DynamoDbEntityStorageConnector._PARTITION_KEY} = :${DynamoDbEntityStorageConnector._PARTITION_KEY}`,
+                    ExpressionAttributeNames: {
+                        [`#${DynamoDbEntityStorageConnector._PARTITION_KEY}`]: DynamoDbEntityStorageConnector._PARTITION_KEY
+                    },
+                    ExpressionAttributeValues: {
+                        [`:${DynamoDbEntityStorageConnector._PARTITION_KEY}`]: { S: partitionKey }
+                    },
+                    Limit: batchSize,
+                    ExclusiveStartKey: exclusiveStartKey
+                }));
+                exclusiveStartKey = lastKey;
+                if (Is.arrayValue(items)) {
+                    for (let j = 0; j < items.length; j += chunkSize) {
+                        const chunk = items.slice(j, j + chunkSize);
+                        await dbConnection.send(new BatchWriteItemCommand({
+                            RequestItems: {
+                                [destConnector._config.tableName]: chunk.map(item => ({
+                                    PutRequest: { Item: item }
+                                }))
+                            }
+                        }));
+                    }
+                }
+            } while (exclusiveStartKey);
+        }
+    }
     /**
      * Create the parameters for a query.
      * @param objectPath The path for the nested object.
@@ -618,19 +788,44 @@ export class DynamoDbEntityStorageConnector {
         if (Is.undefined(condition)) {
             return {
                 keyCondition: "",
-                filterCondition: ""
+                filterCondition: "",
+                requiresScan: false
             };
         }
         if ("conditions" in condition) {
             if (condition.conditions.length === 0) {
                 return {
                     keyCondition: "",
-                    filterCondition: ""
+                    filterCondition: "",
+                    requiresScan: false
                 };
             }
             // It's a group of comparisons, so check the individual items and combine with the logical operator
             const joinConditions = condition.conditions.map(c => this.buildQueryParameters(objectPath, c, attributeNames, attributeValues, secondaryIndex));
             const logicalOperator = this.mapConditionalOperator(condition.logicalOperator);
+            // DynamoDB does not support OR in KeyConditionExpression, so when the operator
+            // is OR we must move all conditions (including key conditions) into FilterExpression.
+            if (condition.logicalOperator === LogicalOperator.Or) {
+                const parts = joinConditions
+                    .map(j => {
+                    const subParts = [j.keyCondition.trim(), j.filterCondition.trim()].filter(s => s.length > 0);
+                    if (subParts.length === 0) {
+                        return "";
+                    }
+                    if (subParts.length === 1) {
+                        return subParts[0];
+                    }
+                    return `(${subParts.join(" AND ")})`;
+                })
+                    .filter(s => s.length > 0);
+                const hasKeyConditions = joinConditions.some(j => j.keyCondition.length > 0);
+                const filterCondition = parts.join(" OR ");
+                return {
+                    keyCondition: "",
+                    filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : "",
+                    requiresScan: hasKeyConditions
+                };
+            }
             const keyCondition = joinConditions
                 .filter(j => j.keyCondition.length > 0)
                 .map(j => j.keyCondition)
@@ -641,7 +836,8 @@ export class DynamoDbEntityStorageConnector {
                 .join(` ${logicalOperator} `);
             return {
                 keyCondition: Is.stringValue(keyCondition) ? ` (${keyCondition}) ` : "",
-                filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : ""
+                filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : "",
+                requiresScan: joinConditions.some(j => j.requiresScan)
             };
         }
         const schemaProp = this._entitySchema.properties?.find(p => p.property === condition.property);
@@ -650,7 +846,8 @@ export class DynamoDbEntityStorageConnector {
         const isKey = schemaProp?.isPrimary ?? (schemaProp?.isSecondary && schemaProp?.property === secondaryIndex);
         return {
             keyCondition: isKey ? comparison : "",
-            filterCondition: !isKey ? comparison : ""
+            filterCondition: !isKey ? comparison : "",
+            requiresScan: false
         };
     }
     /**
@@ -672,14 +869,24 @@ export class DynamoDbEntityStorageConnector {
         prop += comparator.property;
         let attributeName = this.populateAttributeNames(prop, attributeNames);
         if (Is.empty(comparator.value)) {
+            // prepareEntity converts undefined → null before storing, so DynamoDB holds the
+            // attribute with type NULL rather than omitting it. Use attribute_type to match.
+            const nullTypePropName = `:${attributeName.replace(/\./g, "").replace(/#/g, "")}Null`;
+            attributeValues[nullTypePropName] = { S: "NULL" };
             if (comparator.comparison === ComparisonOperator.Equals) {
-                return `attribute_not_exists(${attributeName})`;
+                return `attribute_type(${attributeName}, ${nullTypePropName})`;
             }
             else if (comparator.comparison === ComparisonOperator.NotEquals) {
-                return `attribute_exists(${attributeName})`;
+                return `NOT attribute_type(${attributeName}, ${nullTypePropName})`;
             }
         }
-        let propName = `:${attributeName.replace(/\./g, "").replace(/#/g, "")}`;
+        const basePropName = `:${attributeName.replace(/\./g, "").replace(/#/g, "")}`;
+        let propName = basePropName;
+        let propSuffix = 0;
+        while (!Is.undefined(attributeValues[propName])) {
+            propSuffix++;
+            propName = `${basePropName}${propSuffix}`;
+        }
         if (Is.array(comparator.value)) {
             const dbValues = comparator.value.map(v => this.propertyToDbValue(v, type));
             const arrAttributeNames = [];
@@ -716,7 +923,7 @@ export class DynamoDbEntityStorageConnector {
             return `contains(${attributeName}, ${propName})`;
         }
         else if (comparator.comparison === ComparisonOperator.NotIncludes) {
-            return `notContains(${attributeName}, ${propName})`;
+            return `NOT contains(${attributeName}, ${propName})`;
         }
         else if (comparator.comparison === ComparisonOperator.In) {
             return `${propName} IN ${attributeName}`;
@@ -816,9 +1023,9 @@ export class DynamoDbEntityStorageConnector {
      * @internal
      */
     createConnectionConfig() {
-        const requestHandler = {
-            requestTimeout: this._config.connectionTimeoutMs
-        };
+        const requestHandler = Is.number(this._config.connectionTimeoutMs)
+            ? { requestTimeout: this._config.connectionTimeoutMs }
+            : undefined;
         if (Is.stringValue(this._config.secretAccessKey) &&
             Is.stringValue(this._config.accessKeyId) &&
             this._config.authMode === "credentials") {
@@ -829,13 +1036,15 @@ export class DynamoDbEntityStorageConnector {
                 },
                 endpoint: this._config.endpoint,
                 region: this._config.region,
-                requestHandler
+                requestHandler,
+                maxAttempts: this._config.maxAttempts
             };
         }
         return {
             endpoint: this._config.endpoint,
             region: this._config.region,
-            requestHandler
+            requestHandler,
+            maxAttempts: this._config.maxAttempts
         };
     }
     /**
@@ -847,8 +1056,9 @@ export class DynamoDbEntityStorageConnector {
     async tableExists(tableName) {
         try {
             const dbConnection = this.createConnection();
-            await dbConnection.describeTable({ TableName: tableName });
-            return true;
+            const result = await dbConnection.describeTable({ TableName: tableName });
+            // A table in DELETING state should not be treated as existing
+            return result.Table?.TableStatus !== "DELETING";
         }
         catch {
             return false;
@@ -903,6 +1113,49 @@ export class DynamoDbEntityStorageConnector {
                 }
             };
             const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues, secondaryIndex);
+            // OR conditions on primary key attributes can't use KeyConditionExpression or
+            // FilterExpression in a QueryCommand — fall back to a full table ScanCommand.
+            if (expressions.requiresScan) {
+                let scanFilter = "#partitionId = :partitionId";
+                if (Is.stringValue(expressions.filterCondition)) {
+                    scanFilter += ` AND ${expressions.filterCondition.trim()}`;
+                }
+                const dbConnection = this.createConnection();
+                const matchingItems = [];
+                let scanStartKey = Is.empty(cursor)
+                    ? undefined
+                    : ObjectHelper.fromBytes(Converter.base64ToBytes(cursor));
+                do {
+                    const scanResult = await dbConnection.send(new RawScanCommand({
+                        TableName: this._config.tableName,
+                        FilterExpression: scanFilter,
+                        ExpressionAttributeNames: attributeNames,
+                        ExpressionAttributeValues: attributeValues,
+                        ProjectionExpression: properties?.map(p => p).join(", "),
+                        ExclusiveStartKey: scanStartKey
+                    }));
+                    matchingItems.push(...(scanResult.Items ?? []));
+                    scanStartKey = scanResult.LastEvaluatedKey;
+                } while (!Is.empty(scanStartKey));
+                const hasMore = matchingItems.length > returnSize;
+                const returnedRawItems = hasMore ? matchingItems.slice(0, returnSize) : matchingItems;
+                let resultCursor;
+                if (hasMore) {
+                    const lastRawItem = returnedRawItems[returnedRawItems.length - 1];
+                    const syntheticKey = {
+                        [DynamoDbEntityStorageConnector._PARTITION_KEY]: lastRawItem[DynamoDbEntityStorageConnector._PARTITION_KEY],
+                        [this._primaryKey.property]: lastRawItem[this._primaryKey.property]
+                    };
+                    resultCursor = Converter.bytesToBase64(ObjectHelper.toBytes(syntheticKey));
+                }
+                const scanEntities = returnedRawItems.map(item => {
+                    const unmarshalled = unmarshall(item);
+                    return EntityHelper.unPrepareEntity(unmarshalled, [
+                        DynamoDbEntityStorageConnector._PARTITION_KEY
+                    ]);
+                });
+                return { entities: scanEntities, cursor: resultCursor };
+            }
             let keyExpression = "#partitionId = :partitionId";
             if (expressions.keyCondition.length > 0) {
                 keyExpression += ` AND ${expressions.keyCondition}`;
@@ -917,7 +1170,7 @@ export class DynamoDbEntityStorageConnector {
                 ExpressionAttributeNames: attributeNames,
                 ExpressionAttributeValues: attributeValues,
                 ProjectionExpression: properties?.map(p => p).join(", "),
-                Limit: returnSize,
+                Limit: returnSize + 1,
                 ScanIndexForward: scanAscending,
                 ExclusiveStartKey: Is.empty(cursor)
                     ? undefined
@@ -925,20 +1178,28 @@ export class DynamoDbEntityStorageConnector {
             });
             const connection = this.createDocClient();
             const results = await connection.send(query);
-            let entities = [];
-            if (Is.arrayValue(results.Items)) {
-                entities = results.Items.map(item => {
-                    const unmarshalled = unmarshall(item);
-                    delete unmarshalled[DynamoDbEntityStorageConnector._PARTITION_KEY];
-                    return unmarshalled;
-                });
+            const rawItems = results.Items ?? [];
+            const hasMore = rawItems.length > returnSize;
+            const returnedRawItems = hasMore ? rawItems.slice(0, returnSize) : rawItems;
+            let resultCursor;
+            if (hasMore) {
+                const lastRawItem = returnedRawItems[returnedRawItems.length - 1];
+                const syntheticKey = {
+                    [DynamoDbEntityStorageConnector._PARTITION_KEY]: lastRawItem[DynamoDbEntityStorageConnector._PARTITION_KEY],
+                    [this._primaryKey.property]: lastRawItem[this._primaryKey.property]
+                };
+                if (Is.stringValue(secondaryIndex)) {
+                    syntheticKey[secondaryIndex] = lastRawItem[secondaryIndex];
+                }
+                resultCursor = Converter.bytesToBase64(ObjectHelper.toBytes(syntheticKey));
             }
-            return {
-                entities,
-                cursor: Is.empty(results.LastEvaluatedKey)
-                    ? undefined
-                    : Converter.bytesToBase64(ObjectHelper.toBytes(results.LastEvaluatedKey))
-            };
+            const entities = returnedRawItems.map(item => {
+                const unmarshalled = unmarshall(item);
+                return EntityHelper.unPrepareEntity(unmarshalled, [
+                    DynamoDbEntityStorageConnector._PARTITION_KEY
+                ]);
+            });
+            return { entities, cursor: resultCursor };
         }
         catch (err) {
             if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {