@twin.org/entity-storage-connector-dynamodb 0.0.1-next.2

package/dist/esm/index.mjs

@@ -0,0 +1,645 @@
+import { waitUntilTableExists, QueryCommand, DynamoDB } from '@aws-sdk/client-dynamodb';
+import { GetCommand, PutCommand, DeleteCommand, DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';
+import { unmarshall } from '@aws-sdk/util-dynamodb';
+import { Guards, Is, BaseError, GeneralError, ObjectHelper, Converter, Coerce } from '@twin.org/core';
+import { EntitySchemaFactory, EntitySchemaHelper, ComparisonOperator, LogicalOperator } from '@twin.org/entity';
+import { LoggingConnectorFactory } from '@twin.org/logging-models';
+
+// Copyright 2024 IOTA Stiftung.
+// SPDX-License-Identifier: Apache-2.0.
+/**
+ * Class for performing entity storage operations using Dynamo DB.
+ */
+class DynamoDbEntityStorageConnector {
+    /**
+     * Limit the number of entities when finding.
+     * @internal
+     */
+    static _PAGE_SIZE = 40;
+    /**
+     * Partition id field name.
+     * @internal
+     */
+    static _PARTITION_ID_NAME = "partitionId";
+    /**
+     * Partition id field value.
+     * @internal
+     */
+    static _PARTITION_ID_VALUE = "1";
+    /**
+     * Runtime name for the class.
+     */
+    CLASS_NAME = "DynamoDbEntityStorageConnector";
+    /**
+     * The schema for the entity.
+     * @internal
+     */
+    _entitySchema;
+    /**
+     * The primary key.
+     * @internal
+     */
+    _primaryKey;
+    /**
+     * The configuration for the connector.
+     * @internal
+     */
+    _config;
+    /**
+     * Create a new instance of DynamoDbEntityStorageConnector.
+     * @param options The options for the connector.
+     * @param options.entitySchema The schema for the entity.
+     * @param options.loggingConnectorType The type of logging connector to use, defaults to no logging.
+     * @param options.config The configuration for the connector.
+     */
+    constructor(options) {
+        Guards.object(this.CLASS_NAME, "options", options);
+        Guards.stringValue(this.CLASS_NAME, "options.entitySchema", options.entitySchema);
+        Guards.object(this.CLASS_NAME, "options.config", options.config);
+        Guards.stringValue(this.CLASS_NAME, "options.config.accessKeyId", options.config.accessKeyId);
+        Guards.stringValue(this.CLASS_NAME, "options.config.secretAccessKey", options.config.secretAccessKey);
+        Guards.stringValue(this.CLASS_NAME, "options.config.region", options.config.region);
+        Guards.stringValue(this.CLASS_NAME, "options.config.tableName", options.config.tableName);
+        this._entitySchema = EntitySchemaFactory.get(options.entitySchema);
+        this._primaryKey = EntitySchemaHelper.getPrimaryKey(this._entitySchema);
+        this._config = options.config;
+        this._config.endpoint = Is.stringValue(this._config.endpoint)
+            ? this._config.endpoint
+            : undefined;
+    }
+    /**
+     * Bootstrap the component by creating and initializing any resources it needs.
+     * @param nodeLoggingConnectorType The node logging connector type, defaults to "node-logging".
+     * @returns True if the bootstrapping process was successful.
+     */
+    async bootstrap(nodeLoggingConnectorType) {
+        const nodeLogging = LoggingConnectorFactory.getIfExists(nodeLoggingConnectorType ?? "node-logging");
+        if (!(await this.tableExists(this._config.tableName))) {
+            await nodeLogging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                ts: Date.now(),
+                message: "tableCreating",
+                data: {
+                    tableName: this._config.tableName
+                }
+            });
+            try {
+                const dbConnection = this.createConnection();
+                const tableParams = {
+                    AttributeDefinitions: [],
+                    KeySchema: [],
+                    ProvisionedThroughput: {
+                        ReadCapacityUnits: 1,
+                        WriteCapacityUnits: 1
+                    },
+                    TableName: this._config.tableName
+                };
+                // We always add a partition key to the table as a non optional hash key
+                // is always required when querying using sort parameters
+                tableParams.AttributeDefinitions?.push({
+                    AttributeName: DynamoDbEntityStorageConnector._PARTITION_ID_NAME,
+                    AttributeType: "S"
+                });
+                tableParams.KeySchema?.push({
+                    AttributeName: DynamoDbEntityStorageConnector._PARTITION_ID_NAME,
+                    KeyType: "HASH"
+                });
+                const gsi = [];
+                if (Is.arrayValue(this._entitySchema.properties)) {
+                    for (const prop of this._entitySchema.properties) {
+                        if (prop.isPrimary) {
+                            tableParams.AttributeDefinitions?.push({
+                                AttributeName: prop.property,
+                                AttributeType: prop.type === "integer" || prop.type === "number" ? "N" : "S"
+                            });
+                            tableParams.KeySchema?.push({
+                                AttributeName: prop.property,
+                                KeyType: "RANGE"
+                            });
+                        }
+                        else if (Is.stringValue(prop.sortDirection) || prop.isSecondary) {
+                            // You can only query and sort items if you have a secondary index
+                            // defined for the property
+                            tableParams.AttributeDefinitions?.push({
+                                AttributeName: prop.property,
+                                AttributeType: prop.type === "integer" || prop.type === "number" ? "N" : "S"
+                            });
+                            gsi.push({
+                                IndexName: `${prop.property}Index`,
+                                KeySchema: [
+                                    {
+                                        AttributeName: DynamoDbEntityStorageConnector._PARTITION_ID_NAME,
+                                        KeyType: "HASH"
+                                    },
+                                    {
+                                        AttributeName: prop.property,
+                                        KeyType: "RANGE"
+                                    }
+                                ],
+                                Projection: {
+                                    ProjectionType: "ALL"
+                                },
+                                ProvisionedThroughput: {
+                                    ReadCapacityUnits: 1,
+                                    WriteCapacityUnits: 1
+                                }
+                            });
+                        }
+                    }
+                }
+                if (gsi.length > 0) {
+                    tableParams.GlobalSecondaryIndexes = gsi;
+                }
+                await dbConnection.createTable(tableParams);
+                // Wait for table to exist
+                await waitUntilTableExists({
+                    client: dbConnection,
+                    maxWaitTime: 60000
+                }, {
+                    TableName: this._config.tableName
+                });
+                await nodeLogging?.log({
+                    level: "info",
+                    source: this.CLASS_NAME,
+                    ts: Date.now(),
+                    message: "tableCreated",
+                    data: {
+                        tableName: this._config.tableName
+                    }
+                });
+            }
+            catch (err) {
+                if (BaseError.isErrorCode(err, "ResourceInUseException")) {
+                    await nodeLogging?.log({
+                        level: "info",
+                        source: this.CLASS_NAME,
+                        ts: Date.now(),
+                        message: "tableExists",
+                        data: {
+                            tableName: this._config.tableName
+                        }
+                    });
+                }
+                else {
+                    const errors = err instanceof AggregateError ? err.errors : [err];
+                    for (const error of errors) {
+                        await nodeLogging?.log({
+                            level: "error",
+                            source: this.CLASS_NAME,
+                            ts: Date.now(),
+                            message: "tableCreateFailed",
+                            error: BaseError.fromError(error),
+                            data: {
+                                tableName: this._config.tableName
+                            }
+                        });
+                    }
+                }
+                return false;
+            }
+        }
+        else {
+            await nodeLogging?.log({
+                level: "info",
+                source: this.CLASS_NAME,
+                ts: Date.now(),
+                message: "tableExists",
+                data: {
+                    tableName: this._config.tableName
+                }
+            });
+        }
+        return true;
+    }
+    /**
+     * Get an entity.
+     * @param id The id of the entity to get, or the index value if secondaryIndex is set.
+     * @param secondaryIndex Get the item using a secondary index.
+     * @returns The object if it can be found or undefined.
+     */
+    async get(id, secondaryIndex) {
+        Guards.stringValue(this.CLASS_NAME, "id", id);
+        try {
+            const docClient = this.createDocClient();
+            if (Is.undefined(secondaryIndex)) {
+                const getCommand = new GetCommand({
+                    TableName: this._config.tableName,
+                    Key: {
+                        [DynamoDbEntityStorageConnector._PARTITION_ID_NAME]: DynamoDbEntityStorageConnector._PARTITION_ID_VALUE,
+                        [this._primaryKey.property]: id
+                    }
+                });
+                const response = await docClient.send(getCommand);
+                delete response.Item?.[DynamoDbEntityStorageConnector._PARTITION_ID_NAME];
+                return response.Item;
+            }
+            const secIndex = secondaryIndex.toString();
+            const globalSecondaryIndex = `${secIndex}Index`;
+            const queryCommand = new QueryCommand({
+                TableName: this._config.tableName,
+                IndexName: globalSecondaryIndex,
+                KeyConditionExpression: `#${secIndex} = :id AND #${DynamoDbEntityStorageConnector._PARTITION_ID_NAME} = :${DynamoDbEntityStorageConnector._PARTITION_ID_NAME}`,
+                ExpressionAttributeNames: {
+                    [`#${secIndex}`]: secIndex,
+                    [`#${DynamoDbEntityStorageConnector._PARTITION_ID_NAME}`]: DynamoDbEntityStorageConnector._PARTITION_ID_NAME
+                },
+                ExpressionAttributeValues: {
+                    [`:${DynamoDbEntityStorageConnector._PARTITION_ID_NAME}`]: {
+                        S: DynamoDbEntityStorageConnector._PARTITION_ID_VALUE
+                    },
+                    ":id": { S: id }
+                }
+            });
+            const response = await docClient.send(queryCommand);
+            if (response.Items?.length === 1) {
+                return unmarshall(response.Items[0]);
+            }
+        }
+        catch (err) {
+            if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
+                throw new GeneralError(this.CLASS_NAME, "tableDoesNotExist", {
+                    table: this._config.tableName
+                }, err);
+            }
+            throw new GeneralError(this.CLASS_NAME, "getFailed", {
+                id
+            }, err);
+        }
+        return undefined;
+    }
+    /**
+     * Set an entity.
+     * @param entity The entity to set.
+     * @returns The id of the entity.
+     */
+    async set(entity) {
+        Guards.object(this.CLASS_NAME, "entity", entity);
+        const id = entity[this._primaryKey.property];
+        try {
+            const docClient = this.createDocClient();
+            const putCommand = new PutCommand({
+                TableName: this._config.tableName,
+                Item: {
+                    [DynamoDbEntityStorageConnector._PARTITION_ID_NAME]: DynamoDbEntityStorageConnector._PARTITION_ID_VALUE,
+                    ...entity
+                }
+            });
+            await docClient.send(putCommand);
+        }
+        catch (err) {
+            if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
+                throw new GeneralError(this.CLASS_NAME, "tableDoesNotExist", {
+                    tableName: this._config.tableName
+                }, err);
+            }
+            throw new GeneralError(this.CLASS_NAME, "setFailed", {
+                id
+            }, err);
+        }
+    }
+    /**
+     * Remove the entity.
+     * @param id The id of the entity to remove.
+     * @returns Nothing.
+     */
+    async remove(id) {
+        Guards.stringValue(this.CLASS_NAME, "id", id);
+        try {
+            const docClient = this.createDocClient();
+            const deleteCommand = new DeleteCommand({
+                TableName: this._config.tableName,
+                Key: {
+                    [DynamoDbEntityStorageConnector._PARTITION_ID_NAME]: DynamoDbEntityStorageConnector._PARTITION_ID_VALUE,
+                    [this._primaryKey.property]: id
+                }
+            });
+            await docClient.send(deleteCommand);
+        }
+        catch (err) {
+            if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
+                throw new GeneralError(this.CLASS_NAME, "tableDoesNotExist", {
+                    table: this._config.tableName
+                }, err);
+            }
+            throw new GeneralError(this.CLASS_NAME, "removeFailed", {
+                id
+            }, err);
+        }
+    }
+    /**
+     * Find all the entities which match the conditions.
+     * @param conditions The conditions to match for the entities.
+     * @param sortProperties The optional sort order.
+     * @param properties The optional properties to return, defaults to all.
+     * @param cursor The cursor to request the next page of entities.
+     * @param pageSize The suggested number of entities to return in each chunk, in some scenarios can return a different amount.
+     * @returns All the entities for the storage matching the conditions,
+     * and a cursor which can be used to request more entities.
+     */
+    async query(conditions, sortProperties, properties, cursor, pageSize) {
+        const sql = "";
+        try {
+            const returnSize = pageSize ?? DynamoDbEntityStorageConnector._PAGE_SIZE;
+            let indexName;
+            // If we have a sortable property defined in the descriptor then we must use
+            // the secondary index for the query
+            if (Is.arrayValue(sortProperties)) {
+                if (sortProperties.length > 1) {
+                    throw new GeneralError(this.CLASS_NAME, "sortSingle");
+                }
+                for (const sortProperty of sortProperties) {
+                    const propertySchema = this._entitySchema.properties?.find(e => e.property === sortProperty.property);
+                    if (Is.undefined(propertySchema) ||
+                        (!propertySchema.isPrimary &&
+                            !propertySchema.isSecondary &&
+                            Is.empty(propertySchema.sortDirection))) {
+                        throw new GeneralError(this.CLASS_NAME, "sortNotIndexed", {
+                            property: sortProperty.property
+                        });
+                    }
+                    indexName = propertySchema.isPrimary
+                        ? undefined
+                        : `${sortProperty.property}Index`;
+                }
+            }
+            const attributeNames = { "#partitionId": "partitionId" };
+            const attributeValues = {
+                [`:${DynamoDbEntityStorageConnector._PARTITION_ID_NAME}`]: {
+                    S: DynamoDbEntityStorageConnector._PARTITION_ID_VALUE
+                }
+            };
+            const expressions = this.buildQueryParameters("", conditions, attributeNames, attributeValues);
+            let keyExpression = "#partitionId = :partitionId";
+            if (expressions.keyCondition.length > 0) {
+                keyExpression += ` AND ${expressions.keyCondition}`;
+            }
+            const query = new QueryCommand({
+                TableName: this._config.tableName,
+                IndexName: indexName,
+                KeyConditionExpression: keyExpression,
+                FilterExpression: Is.stringValue(expressions.filterCondition)
+                    ? expressions.filterCondition
+                    : undefined,
+                ExpressionAttributeNames: attributeNames,
+                ExpressionAttributeValues: attributeValues,
+                ProjectionExpression: properties?.map(p => p).join(", "),
+                Limit: returnSize,
+                ExclusiveStartKey: Is.empty(cursor)
+                    ? undefined
+                    : ObjectHelper.fromBytes(Converter.base64ToBytes(cursor))
+            });
+            const connection = this.createDocClient();
+            const results = await connection.send(query);
+            let entities = [];
+            if (Is.arrayValue(results.Items)) {
+                entities = results.Items.map(item => unmarshall(item));
+            }
+            return {
+                entities,
+                cursor: Is.empty(results.LastEvaluatedKey)
+                    ? undefined
+                    : Converter.bytesToBase64(ObjectHelper.toBytes(results.LastEvaluatedKey))
+            };
+        }
+        catch (err) {
+            if (BaseError.isErrorCode(err, "ResourceNotFoundException")) {
+                throw new GeneralError(this.CLASS_NAME, "tableDoesNotExist", {
+                    table: this._config.tableName
+                }, err);
+            }
+            throw new GeneralError(this.CLASS_NAME, "queryFailed", {
+                sql
+            }, err);
+        }
+    }
+    /**
+     * Delete the table.
+     * @returns Nothing.
+     */
+    async tableDelete() {
+        try {
+            const dbConnection = this.createConnection();
+            await dbConnection.deleteTable({ TableName: this._config.tableName });
+        }
+        catch { }
+    }
+    /**
+     * Create an SQL condition clause.
+     * @param objectPath The path for the nested object.
+     * @param condition The conditions to create the query from.
+     * @param attributeNames The attribute names to use in the query.
+     * @param attributeValues The attribute values to use in the query.
+     * @returns The condition clause.
+     * @internal
+     */
+    buildQueryParameters(objectPath, condition, attributeNames, attributeValues) {
+        // If no conditions are defined then return empty string
+        if (Is.undefined(condition)) {
+            return {
+                keyCondition: "",
+                filterCondition: ""
+            };
+        }
+        if ("conditions" in condition) {
+            // It's a group of comparisons, so check the individual items and combine with the logical operator
+            const joinConditions = condition.conditions.map(c => this.buildQueryParameters(objectPath, c, attributeNames, attributeValues));
+            const logicalOperator = this.mapConditionalOperator(condition.logicalOperator);
+            const keyCondition = joinConditions.map(j => j.keyCondition).join(` ${logicalOperator} `);
+            const filterCondition = joinConditions
+                .map(j => j.filterCondition)
+                .join(` ${logicalOperator} `);
+            return {
+                keyCondition: Is.stringValue(keyCondition) ? ` (${keyCondition}) ` : "",
+                filterCondition: Is.stringValue(filterCondition) ? ` (${filterCondition}) ` : ""
+            };
+        }
+        const schemaProp = this._entitySchema.properties?.find(p => p.property === condition.property);
+        // It's a single value so just create the property comparison for the condition
+        const comparison = this.mapComparisonOperator(objectPath, condition, schemaProp?.type, attributeNames, attributeValues);
+        return {
+            keyCondition: schemaProp?.isPrimary ? comparison : "",
+            filterCondition: schemaProp?.isPrimary ? "" : comparison
+        };
+    }
+    /**
+     * Map the framework comparison operators to those in DynamoDB.
+     * @param objectPath The prefix to use for the condition.
+     * @param comparator The operator to map.
+     * @param type The type of the property.
+     * @param attributeNames The attribute names to use in the query.
+     * @param attributeValues The attribute values to use in the query.
+     * @returns The comparison expression.
+     * @throws GeneralError if the comparison operator is not supported.
+     * @internal
+     */
+    mapComparisonOperator(objectPath, comparator, type, attributeNames, attributeValues) {
+        let prop = objectPath;
+        if (prop.length > 0) {
+            prop += ".";
+        }
+        prop += comparator.property;
+        let attributeName = this.populateAttributeNames(prop, attributeNames);
+        let propName = `:${attributeName.replace(/\./g, "").replace(/#/g, "")}`;
+        if (Is.array(comparator.value)) {
+            const dbValues = comparator.value.map(v => this.propertyToDbValue(v, type));
+            const arrAttributeNames = [];
+            for (let i = 0; i < dbValues.length; i++) {
+                const arrAttributeName = `${propName}${i}`;
+                attributeValues[arrAttributeName] = dbValues[i];
+                arrAttributeNames.push(arrAttributeName);
+            }
+            propName = attributeName;
+            attributeName = `(${arrAttributeNames.join(", ")})`;
+        }
+        else {
+            attributeValues[propName] = this.propertyToDbValue(comparator.value, type);
+        }
+        if (comparator.comparison === ComparisonOperator.Equals) {
+            return `${attributeName} = ${propName}`;
+        }
+        else if (comparator.comparison === ComparisonOperator.NotEquals) {
+            return `${attributeName} <> ${propName}`;
+        }
+        else if (comparator.comparison === ComparisonOperator.GreaterThan) {
+            return `${attributeName} > ${propName}`;
+        }
+        else if (comparator.comparison === ComparisonOperator.LessThan) {
+            return `${attributeName} < ${propName}`;
+        }
+        else if (comparator.comparison === ComparisonOperator.GreaterThanOrEqual) {
+            return `${attributeName} >= ${propName}`;
+        }
+        else if (comparator.comparison === ComparisonOperator.LessThanOrEqual) {
+            return `${attributeName} <= ${propName}`;
+        }
+        else if (comparator.comparison === ComparisonOperator.Includes) {
+            return `contains(${attributeName}, ${propName})`;
+        }
+        else if (comparator.comparison === ComparisonOperator.NotIncludes) {
+            return `notContains(${attributeName}, ${propName})`;
+        }
+        else if (comparator.comparison === ComparisonOperator.In) {
+            return `${propName} IN ${attributeName}`;
+        }
+        throw new GeneralError(this.CLASS_NAME, "comparisonNotSupported", {
+            comparison: comparator.comparison
+        });
+    }
+    /**
+     * Create a unique name for the attribute.
+     * @param name The name to create a unique name for.
+     * @param attributeNames The attribute names to use in the query.
+     * @returns The unique name.
+     * @internal
+     */
+    populateAttributeNames(name, attributeNames) {
+        const parts = name.split(".");
+        const attributeNameParts = [];
+        for (const part of parts) {
+            const hashPart = `#${part}`;
+            if (Is.empty(attributeNames[hashPart])) {
+                attributeNames[hashPart] = part;
+            }
+            attributeNameParts.push(hashPart);
+        }
+        return attributeNameParts.join(".");
+    }
+    /**
+     * Map the framework conditional operators to those in DynamoDB.
+     * @param operator The operator to map.
+     * @returns The conditional operator.
+     * @throws GeneralError if the conditional operator is not supported.
+     * @internal
+     */
+    mapConditionalOperator(operator) {
+        if ((operator ?? LogicalOperator.And) === LogicalOperator.And) {
+            return "AND";
+        }
+        else if (operator === LogicalOperator.Or) {
+            return "OR";
+        }
+        throw new GeneralError(this.CLASS_NAME, "conditionalNotSupported", { operator });
+    }
+    /**
+     * Format a value to insert into DB.
+     * @param value The value to format.
+     * @param type The type for the property.
+     * @returns The value after conversion.
+     * @internal
+     */
+    propertyToDbValue(value, type) {
+        if (Is.object(value)) {
+            const map = {};
+            for (const key in value) {
+                map[key] = this.propertyToDbValue(value[key]);
+            }
+            return {
+                M: map
+            };
+        }
+        if (type === "integer" || type === "number") {
+            return { N: Coerce.string(value) ?? "" };
+        }
+        else if (type === "boolean") {
+            return { BOOL: Coerce.boolean(value) ?? false };
+        }
+        return { S: Coerce.string(value) ?? "" };
+    }
+    /**
+     * Create a doc client connection.
+     * @returns The dynamo db document client.
+     * @internal
+     */
+    createDocClient() {
+        return DynamoDBDocumentClient.from(new DynamoDB({
+            apiVersion: "2012-10-08",
+            ...this.createConnectionConfig()
+        }), {
+            marshallOptions: {
+                removeUndefinedValues: true
+            }
+        });
+    }
+    /**
+     * Create a new DB connection.
+     * @returns The dynamo db connection.
+     * @internal
+     */
+    createConnection() {
+        return new DynamoDB(this.createConnectionConfig());
+    }
+    /**
+     * Create a new DB connection configuration.
+     * @returns The dynamo db connection configuration.
+     * @internal
+     */
+    createConnectionConfig() {
+        return {
+            credentials: {
+                accessKeyId: this._config.accessKeyId,
+                secretAccessKey: this._config.secretAccessKey
+            },
+            endpoint: this._config.endpoint,
+            region: this._config.region
+        };
+    }
+    /**
+     * Check if the table exists.
+     * @param tableName The table to check.
+     * @returns True if the table exists.
+     * @internal
+     */
+    async tableExists(tableName) {
+        try {
+            const dbConnection = this.createConnection();
+            await dbConnection.describeTable({ TableName: tableName });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+}
+
+export { DynamoDbEntityStorageConnector };