@webda/aws 4.0.0-beta.1 → 4.0.0-beta.3

package/lib/services/dynamodb.js

@@ -1,711 +0,0 @@
-import { ConditionalCheckFailedException, DynamoDB, DynamoDBClient } from "@aws-sdk/client-dynamodb";
-import { DynamoDBDocument } from "@aws-sdk/lib-dynamodb";
-import { CoreModel, Store, StoreNotFoundError, StoreParameters, UpdateConditionFailError, WebdaError } from "@webda/core";
-import * as WebdaQL from "@webda/ql";
-import { AWSServiceParameters } from "./index.js";
-/**
- * Define DynamoDB parameters
- */
-export class DynamoStoreParameters extends AWSServiceParameters(StoreParameters) {
-    constructor(params, service) {
-        super(params, service);
-        if (this.table === undefined) {
-            throw new WebdaError.CodeError("DYNAMODB_TABLE_PARAMETER_REQUIRED", "Need to define a table at least");
-        }
-        this.globalIndexes ?? (this.globalIndexes = {});
-    }
-}
-/**
- * DynamoStore handles the DynamoDB
- *
- * Parameters:
- *   accessKeyId: '' // try WEBDA_AWS_KEY env variable if not found
- *   secretAccessKey: '' // try WEBDA_AWS_SECRET env variable if not found
- *   table: ''
- *   region: ''
- *
- * @WebdaModda
- */
-export default class DynamoStore extends Store {
-    /**
-     * Load the parameters
-     *
-     * @param params
-     */
-    loadParameters(params) {
-        return new DynamoStoreParameters(params, this);
-    }
-    /**
-     * Create the AWS client
-     */
-    resolve() {
-        super.resolve();
-        this._client = DynamoDBDocument.from(new DynamoDBClient(this.parameters), {
-            marshallOptions: {
-                removeUndefinedValues: true
-            }
-        });
-        return this;
-    }
-    /**
-     * Copy one DynamoDB table to another
-     *
-     * @param output
-     * @param source
-     * @param target
-     */
-    static async copyTable(output, source, target) {
-        let db = new DynamoDB({});
-        let ExclusiveStartKey;
-        let props = await db.describeTable({
-            TableName: source
-        });
-        output.startProgress("copyTable", props.Table.ItemCount, `Copying ${source} to ${target}`);
-        do {
-            let info = await db.scan({
-                TableName: source,
-                ExclusiveStartKey
-            });
-            do {
-                let items = [];
-                while (info.Items.length && items.length < 25) {
-                    items.push(info.Items.shift());
-                }
-                if (!items.length) {
-                    break;
-                }
-                let params = {
-                    RequestItems: {}
-                };
-                params.RequestItems[target] = items.map(Item => ({
-                    PutRequest: { Item }
-                }));
-                await db.batchWriteItem(params);
-                output.incrementProgress(items.length, "copyTable");
-            } while (true);
-            ExclusiveStartKey = info.LastEvaluatedKey;
-        } while (ExclusiveStartKey);
-    }
-    /**
-     * @inheritdoc
-     */
-    async _exists(uid) {
-        // Should use find + limit 1
-        return (await this._get(uid)) !== undefined;
-    }
-    /**
-     * @inheritdoc
-     */
-    async _save(object, uid = object.uuid) {
-        return this._update(object, uid);
-    }
-    /**
-     * @override
-     *
-     * @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html
-     * @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.html
-     */
-    async find(query) {
-        let scan = true;
-        let IndexName = undefined;
-        let result;
-        let primaryKeys = { uuid: null };
-        Object.keys(this.parameters.globalIndexes).forEach(name => {
-            primaryKeys[this.parameters.globalIndexes[name].key] = name;
-        });
-        // We could use PartQL but localstack is not compatible
-        let filter = new WebdaQL.AndExpression([]);
-        let KeyConditionExpression = "";
-        let ExpressionAttributeValues = {};
-        let FilterExpression = [];
-        let ExpressionAttributeNames = {};
-        let indexNode;
-        let sortOrder = "ASC";
-        let processingExpression;
-        if (!(query.filter instanceof WebdaQL.AndExpression)) {
-            processingExpression = new WebdaQL.AndExpression([query.filter]);
-        }
-        else {
-            processingExpression = query.filter;
-        }
-        // Search for the index
-        processingExpression.children.some(child => {
-            if (child instanceof WebdaQL.ComparisonExpression) {
-                // Primary key requires equal operator
-                if (child.operator === "=" && primaryKeys[child.attribute[0]] !== undefined) {
-                    // Query not scan
-                    scan = false;
-                    IndexName = primaryKeys[child.attribute[0]];
-                    indexNode = child;
-                    KeyConditionExpression = `#${child.attribute[0]} = :${child.attribute[0]}`;
-                    ExpressionAttributeNames[`#${child.attribute[0]}`] = child.attribute[0];
-                    ExpressionAttributeValues[`:${child.attribute[0]}`] = child.value;
-                    if (primaryKeys[child.attribute[0]] !== null &&
-                        query.orderBy &&
-                        this.parameters.globalIndexes[primaryKeys[child.attribute[0]]].sort) {
-                        const sortKey = this.parameters.globalIndexes[primaryKeys[child.attribute[0]]].sort;
-                        // Might update sort order
-                        query.orderBy.some(order => {
-                            if (order.field === sortKey) {
-                                sortOrder = order.direction;
-                            }
-                        });
-                    }
-                    return true;
-                }
-            }
-        });
-        let count = 1;
-        // Build the Query/Filter
-        processingExpression.children.forEach(child => {
-            if (child === indexNode) {
-                return;
-            }
-            // Only work on Comparison Node for now
-            if (child instanceof WebdaQL.ComparisonExpression) {
-                let operator = child.operator;
-                // DynamoDB does not manage LIKE
-                if (child.operator === "LIKE") {
-                    filter.children.push(child);
-                    return;
-                }
-                // CONTAINS -> contains (category, :category1)
-                // != is <> in Dynamo
-                if (operator === "!=") {
-                    operator = "<>";
-                }
-                // DynamoDB does not allow more than 100 items in IN
-                if (child.operator === "IN" && child.value.length > 100) {
-                    filter.children.push(child);
-                    return;
-                }
-                // Subfields like team.id needs to be #a1.#a2
-                let attr = `a${count++}`;
-                let fullAttr = `#${attr}`;
-                child.attribute.slice(1).forEach(v => {
-                    let subAttr = `#a${count++}`;
-                    fullAttr += `.${subAttr}`;
-                    ExpressionAttributeNames[subAttr] = v;
-                });
-                ExpressionAttributeNames[`#${attr}`] = child.attribute[0];
-                let valueExpression = `:${attr}`;
-                if (child.operator === "IN") {
-                    // Need to follow `a IN (b, c, d)
-                    valueExpression = "(" + valueExpression;
-                    ExpressionAttributeValues[`:${attr}`] = child.value[0];
-                    child.value.splice(1).forEach(v => {
-                        let subAttr = `:a${count++}`;
-                        valueExpression += `, ${subAttr}`;
-                        ExpressionAttributeValues[subAttr] = v;
-                    });
-                    valueExpression += ")";
-                }
-                else {
-                    // For all other just give the value
-                    ExpressionAttributeValues[`:${attr}`] = child.value;
-                }
-                // Manage CONTAINS
-                if (child.operator === "CONTAINS") {
-                    FilterExpression.push(`contains(${fullAttr}, :${attr})`);
-                    return;
-                }
-                // If this is a sort key
-                if (IndexName && this.parameters.globalIndexes[IndexName].sort === child.attribute[0]) {
-                    // Sort key
-                    KeyConditionExpression += ` AND ${fullAttr} ${operator} ${valueExpression}`;
-                    return;
-                }
-                // Otherwise fallback to normal Filter
-                FilterExpression.push(`${fullAttr} ${operator} ${valueExpression}`);
-            }
-            else {
-                filter.children.push(child);
-            }
-        });
-        if (!Object.keys(ExpressionAttributeNames).length) {
-            ExpressionAttributeNames = undefined;
-        }
-        if (!Object.keys(ExpressionAttributeValues).length) {
-            ExpressionAttributeValues = undefined;
-        }
-        let ExclusiveStartKey = query.continuationToken
-            ? JSON.parse(Buffer.from(query.continuationToken, "base64").toString())
-            : undefined;
-        // Scan if not primary key was provided
-        if (scan) {
-            // SHould log bad query
-            const Items = [];
-            do {
-                result = await this._client.scan({
-                    TableName: this.parameters.table,
-                    FilterExpression: FilterExpression.length ? FilterExpression.join(" AND ") : undefined,
-                    ExclusiveStartKey,
-                    ExpressionAttributeNames,
-                    ExpressionAttributeValues,
-                    Limit: query.limit - Items.length
-                });
-                Items.push(...result.Items);
-                result.Items = Items;
-                ExclusiveStartKey = result.LastEvaluatedKey;
-            } while (Items.length < query.limit && result.LastEvaluatedKey);
-        }
-        else {
-            result = await this._client.query({
-                TableName: this.parameters.table,
-                IndexName,
-                ExclusiveStartKey,
-                KeyConditionExpression,
-                FilterExpression: FilterExpression.length ? FilterExpression.join(" AND ") : undefined,
-                ExpressionAttributeNames,
-                ExpressionAttributeValues,
-                Limit: query.limit,
-                ScanIndexForward: sortOrder === "ASC" ? true : false
-            });
-        }
-        return {
-            results: result.Items.map(c => this.initModel(c)),
-            filter,
-            continuationToken: result.Items.length >= query.limit
-                ? Buffer.from(JSON.stringify(result.LastEvaluatedKey)).toString("base64")
-                : undefined
-        };
-    }
-    /**
-     * Serialize a date for DynamoDB
-     * @param date
-     * @returns
-     */
-    _serializeDate(date) {
-        return JSON.stringify(date).replace(/"/g, "");
-    }
-    /**
-     * Clean object to store in DynamoDB
-     *
-     * @param object
-     * @returns
-     */
-    _cleanObject(object) {
-        if (typeof object !== "object")
-            return object;
-        if (object instanceof Date) {
-            return this._serializeDate(object);
-        }
-        if (object instanceof CoreModel) {
-            object = object.toStoredJSON();
-        }
-        let res;
-        if (object instanceof Array) {
-            res = [];
-        }
-        else {
-            res = {};
-        }
-        for (let i in object) {
-            if (object[i] === "" || i.startsWith("__store")) {
-                continue;
-            }
-            res[i] = this._cleanObject(object[i]);
-        }
-        return res;
-    }
-    /**
-     * @inheritdoc
-     */
-    async _removeAttribute(uuid, attribute, itemWriteCondition, itemWriteConditionField) {
-        let params = {
-            TableName: this.parameters.table,
-            Key: {
-                uuid
-            }
-        };
-        let attrs = {};
-        attrs["#attr"] = attribute;
-        attrs["#lastUpdate"] = "_lastUpdate";
-        params.ExpressionAttributeNames = attrs;
-        params.ConditionExpression = "attribute_exists(#uu)";
-        params.ExpressionAttributeValues = {
-            ":lastUpdate": this._serializeDate(new Date())
-        };
-        params.UpdateExpression = "REMOVE #attr SET #lastUpdate = :lastUpdate";
-        if (itemWriteCondition) {
-            this.setWriteCondition(params, itemWriteCondition, itemWriteConditionField);
-        }
-        else {
-            attrs["#uu"] = "uuid";
-            params.ConditionExpression = "attribute_exists(#uu)";
-        }
-        try {
-            await this._client.update(params);
-        }
-        catch (err) {
-            if (err instanceof ConditionalCheckFailedException) {
-                if (!itemWriteConditionField) {
-                    throw new StoreNotFoundError(uuid, this.getName());
-                }
-                throw new UpdateConditionFailError(uuid, itemWriteConditionField, itemWriteCondition);
-            }
-            throw err;
-        }
-    }
-    /**
-     * @inheritdoc
-     */
-    async _deleteItemFromCollection(uid, prop, index, itemWriteCondition, itemWriteConditionField, updateDate) {
-        let params = {
-            TableName: this.parameters.table,
-            Key: {
-                uuid: uid
-            }
-        };
-        let attrs = {};
-        attrs["#" + prop] = prop;
-        attrs["#lastUpdate"] = "_lastUpdate";
-        params.ExpressionAttributeNames = attrs;
-        params.ExpressionAttributeValues = {
-            ":lastUpdate": this._serializeDate(updateDate)
-        };
-        params.UpdateExpression = "REMOVE #" + prop + "[" + index + "] SET #lastUpdate = :lastUpdate";
-        if (itemWriteCondition) {
-            params.ExpressionAttributeValues[":condValue"] = itemWriteCondition;
-            attrs["#condName"] = prop;
-            attrs["#field"] = itemWriteConditionField;
-            params.ConditionExpression = "#condName[" + index + "].#field = :condValue";
-        }
-        try {
-            await this._client.update(params);
-        }
-        catch (err) {
-            if (err instanceof ConditionalCheckFailedException) {
-                throw new UpdateConditionFailError(uid, itemWriteConditionField, itemWriteCondition);
-            }
-            else if (err.name === "ValidationException") {
-                throw new StoreNotFoundError(uid, this.getName());
-            }
-            throw err;
-        }
-    }
-    /**
-     * @inheritdoc
-     */
-    async _upsertItemToCollection(uuid, prop, item, index, itemWriteCondition, itemWriteConditionField, updateDate) {
-        let params = {
-            TableName: this.parameters.table,
-            Key: {
-                uuid
-            }
-        };
-        let attrValues = {};
-        let attrs = {};
-        attrs["#" + prop] = prop;
-        attrs["#lastUpdate"] = "_lastUpdate";
-        attrValues[":" + prop] = this._cleanObject(item);
-        attrValues[":lastUpdate"] = this._serializeDate(updateDate);
-        attrValues[":uuid"] = uuid;
-        attrs["#uuid"] = this._uuidField;
-        params.ConditionExpression = "#uuid = :uuid";
-        params.ExpressionAttributeValues = attrValues;
-        params.ExpressionAttributeNames = attrs;
-        if (index === undefined) {
-            params.UpdateExpression =
-                "SET #" +
-                    prop +
-                    "= list_append(if_not_exists (#" +
-                    prop +
-                    ", :empty_list),:" +
-                    prop +
-                    "), #lastUpdate = :lastUpdate";
-            attrValues[":" + prop] = [attrValues[":" + prop]];
-            attrValues[":empty_list"] = [];
-        }
-        else {
-            params.UpdateExpression = "SET #" + prop + "[" + index + "] = :" + prop + ", #lastUpdate = :lastUpdate";
-            if (itemWriteCondition) {
-                attrValues[":condValue"] = itemWriteCondition;
-                attrs["#condName"] = prop;
-                attrs["#field"] = itemWriteConditionField;
-                params.ConditionExpression = "#condName[" + index + "].#field = :condValue and #uuid = :uuid";
-            }
-        }
-        try {
-            await this._client.update(params);
-        }
-        catch (err) {
-            if (err instanceof ConditionalCheckFailedException) {
-                throw new UpdateConditionFailError(uuid, itemWriteConditionField, itemWriteCondition);
-            }
-            else if (err.name === "ValidationException") {
-                throw new StoreNotFoundError(uuid, this.getName());
-            }
-            throw err;
-        }
-    }
-    /**
-     * REturn the write condition as string
-     * @param writeCondition
-     * @param field
-     * @returns
-     */
-    setWriteCondition(params, writeCondition, field = "_lastUpdate") {
-        params.ExpressionAttributeNames ?? (params.ExpressionAttributeNames = {});
-        params.ExpressionAttributeValues ?? (params.ExpressionAttributeValues = {});
-        params.ExpressionAttributeNames["#cf"] = field;
-        if (writeCondition instanceof Date) {
-            writeCondition = this._serializeDate(writeCondition);
-        }
-        params.ExpressionAttributeValues[":cf"] = writeCondition;
-        params.ConditionExpression = "#cf = :cf";
-    }
-    /**
-     * @inheritdoc
-     */
-    async _delete(uid, writeCondition, itemWriteConditionField) {
-        let params = {
-            TableName: this.parameters.table,
-            Key: {
-                uuid: uid
-            }
-        };
-        if (writeCondition) {
-            this.setWriteCondition(params, writeCondition, itemWriteConditionField);
-        }
-        try {
-            await this._client.delete(params);
-        }
-        catch (err) {
-            if (err instanceof ConditionalCheckFailedException) {
-                throw new UpdateConditionFailError(uid, itemWriteConditionField, writeCondition);
-            }
-            throw err;
-        }
-    }
-    /**
-     * @inheritdoc
-     */
-    async _patch(object, uid, itemWriteCondition, itemWriteConditionField) {
-        object = this._cleanObject(object);
-        let expr = "SET ";
-        let sep = "";
-        let attrValues = {};
-        let attrs = {};
-        let skipUpdate = true;
-        let i = 1;
-        for (let attr in object) {
-            if (attr === "uuid" || object[attr] === undefined) {
-                continue;
-            }
-            skipUpdate = false;
-            expr += sep + "#a" + i + " = :v" + i;
-            attrValues[":v" + i] = object[attr];
-            attrs["#a" + i] = attr;
-            sep = ",";
-            i++;
-        }
-        if (skipUpdate) {
-            return;
-        }
-        let params = {
-            TableName: this.parameters.table,
-            Key: {
-                uuid: uid
-            },
-            UpdateExpression: expr,
-            ExpressionAttributeValues: attrValues,
-            ExpressionAttributeNames: attrs
-        };
-        // The Write Condition checks the value before writing
-        if (itemWriteCondition) {
-            this.setWriteCondition(params, itemWriteCondition, itemWriteConditionField);
-        }
-        try {
-            await this._client.update(params);
-        }
-        catch (err) {
-            if (err instanceof ConditionalCheckFailedException) {
-                throw new UpdateConditionFailError(uid, itemWriteConditionField, itemWriteCondition);
-            }
-            throw err;
-        }
-    }
-    /**
-     * @inheritdoc
-     */
-    async _update(object, uid, itemWriteCondition, itemWriteConditionField) {
-        object = this._cleanObject(object);
-        object.uuid = uid;
-        let params = {
-            TableName: this.parameters.table,
-            Item: object
-        };
-        // The Write Condition checks the value before writing
-        if (itemWriteCondition) {
-            this.setWriteCondition(params, itemWriteCondition, itemWriteConditionField);
-        }
-        try {
-            await this._client.put(params);
-            return object;
-        }
-        catch (err) {
-            if (err instanceof ConditionalCheckFailedException) {
-                throw new UpdateConditionFailError(uid, itemWriteConditionField, itemWriteCondition);
-            }
-            throw err;
-        }
-    }
-    /**
-     * @inheritdoc
-     */
-    async _scan(items = [], paging = undefined) {
-        let data = await this._client.scan({
-            TableName: this.parameters.table,
-            Limit: this.parameters.scanPage,
-            ExclusiveStartKey: paging
-        });
-        for (let i in data.Items) {
-            items.push(this.initModel(data.Items[i]));
-        }
-        if (data.LastEvaluatedKey) {
-            return this._scan(items, data.LastEvaluatedKey);
-        }
-        return items;
-    }
-    /**
-     * @inheritdoc
-     */
-    async getAll(uids) {
-        if (!uids) {
-            return this._scan([]);
-        }
-        let params = {
-            RequestItems: {}
-        };
-        params["RequestItems"][this.parameters.table] = {
-            Keys: uids.map(value => {
-                return {
-                    uuid: value
-                };
-            })
-        };
-        let result = await this._client.batchGet(params);
-        return result.Responses[this.parameters.table].map(this.initModel, this);
-    }
-    /**
-     * @inheritdoc
-     */
-    async _get(uid, raiseIfNotFound = false) {
-        let params = {
-            TableName: this.parameters.table,
-            Key: {
-                uuid: uid
-            }
-        };
-        let item = (await this._client.get(params)).Item;
-        if (!item) {
-            if (raiseIfNotFound) {
-                throw new StoreNotFoundError(uid, this.getName());
-            }
-            return undefined;
-        }
-        return this.initModel(item);
-    }
-    /**
-     * @inheritdoc
-     */
-    async _incrementAttributes(uid, parameters, updateDate) {
-        let params = {
-            TableName: this.parameters.table,
-            Key: {
-                uuid: uid
-            },
-            UpdateExpression: "SET #ud = :ud ADD ",
-            ConditionExpression: "attribute_exists(#uu)",
-            ExpressionAttributeValues: {
-                ":ud": this._serializeDate(updateDate)
-            },
-            ExpressionAttributeNames: {
-                "#ud": "_lastUpdate",
-                "#uu": "uuid"
-            }
-        };
-        parameters.forEach((p, i) => {
-            params.ExpressionAttributeValues[`:v${i}`] = p.value;
-            params.ExpressionAttributeNames[`#a${i}`] = p.property;
-        });
-        params.UpdateExpression += parameters.map((_, i) => `#a${i} :v${i}`).join(",");
-        try {
-            return await this._client.update(params);
-        }
-        catch (err) {
-            if (err instanceof ConditionalCheckFailedException) {
-                throw new StoreNotFoundError(uid, this.getName());
-            }
-            throw err;
-        }
-    }
-    /**
-     * @inheritdoc
-     */
-    getARNPolicy(accountId) {
-        let region = this.parameters.region || "us-east-1";
-        return {
-            Sid: this.constructor.name + this._name,
-            Effect: "Allow",
-            Action: [
-                "dynamodb:BatchGetItem",
-                "dynamodb:BatchWriteItem",
-                "dynamodb:DeleteItem",
-                "dynamodb:GetItem",
-                "dynamodb:GetRecords",
-                "dynamodb:GetShardIterator",
-                "dynamodb:PutItem",
-                "dynamodb:Query",
-                "dynamodb:Scan",
-                "dynamodb:UpdateItem"
-            ],
-            Resource: ["arn:aws:dynamodb:" + region + ":" + accountId + ":table/" + this.parameters.table]
-        };
-    }
-    /**
-     * @inheritdoc
-     */
-    async __clean() {
-        let params = {
-            TableName: this.parameters.table
-        };
-        let result = await this._client.scan(params);
-        let promises = [];
-        for (let i in result.Items) {
-            promises.push(this._delete(result.Items[i].uuid));
-        }
-        await Promise.all(promises);
-    }
-    /**
-     * @inheritdoc
-     */
-    getCloudFormation(deployer) {
-        if (this.parameters.CloudFormationSkip) {
-            return {};
-        }
-        let resources = {};
-        this.parameters.CloudFormation = this.parameters.CloudFormation || {};
-        this.parameters.CloudFormation.Table = this.parameters.CloudFormation.Table || {};
-        let KeySchema = this.parameters.CloudFormation.KeySchema || [{ KeyType: "HASH", AttributeName: "uuid" }];
-        let AttributeDefinitions = this.parameters.CloudFormation.AttributeDefinitions || [];
-        this.parameters.CloudFormation.Table.BillingMode =
-            this.parameters.CloudFormation.Table.BillingMode || "PAY_PER_REQUEST";
-        AttributeDefinitions.push({ AttributeName: "uuid", AttributeType: "S" });
-        resources[this._name + "DynamoTable"] = {
-            Type: "AWS::DynamoDB::Table",
-            Properties: {
-                ...this.parameters.CloudFormation.Table,
-                TableName: this.parameters.table,
-                KeySchema,
-                AttributeDefinitions,
-                Tags: deployer.getDefaultTags(this.parameters.CloudFormation.Table.Tags)
-            }
-        };
-        // Add any Other resources with prefix of the service
-        return resources;
-    }
-}
-export { DynamoStore };
-//# sourceMappingURL=dynamodb.js.map