@allthings/cloud-toolkit 1.3.0 → 1.4.0

package/README.md

@@ -18,5 +18,16 @@ Standardizes the setup of aws, datadog and other things
 |--------------|----------------------------|------------------------------------------------------|
 | invokeLambda | invocationType<br/>payload | Invokes set lambda with invocations type and payload |
 
+### dynamodb
+| function        | parameters                           | description                                              |
+|-----------------|--------------------------------------|----------------------------------------------------------|
+| batchWriteItems | table<br/>items                      | Creates multiple item in a defined table                 |
+| deleteItem      | table<br/>key<br/>options (optional) | Deletes database item from defined table by given key    |
+| getItem         | table<br/>key<br/>options (optional) | Returns item from Database by defined table and key      |
+| putItem         | table<br/>key                        | Creates one item in defined table                        |
+| queryTable      | table<br/>options (optional)         | Returns result items of query                            |
+| scanTable       | table<br/>options (optional)         | Returns result items of scan                             |
+| updateItem      | table<br/>key<br/>options (optional) | Updates item in database by provided table and given key |
+
 ### logger
 A winston instance will be created with default log levels https://github.com/winstonjs/winston?tab=readme-ov-file#logging

package/dist/lib.cjs.js

@@ -1,5 +1,8 @@
 'use strict';
 
+var clientDynamodb = require('@aws-sdk/client-dynamodb');
+var libDynamodb = require('@aws-sdk/lib-dynamodb');
+var Bottleneck = require('bottleneck');
 var clientLambda = require('@aws-sdk/client-lambda');
 var clientSsm = require('@aws-sdk/client-ssm');
 var aws = require('@aws-sdk/client-ses');
@@ -29,6 +32,189 @@ const DEFAULT_AWS_CONFIGURATION = {
     region: 'eu-central-2',
 };
 
+const splitEvery = (every, list) => Array.from({ length: Math.ceil(list.length / every) }).map((_, count) => list.slice(count * every, count * every + every));
+const until = async (predicate, transformer, initialValue, iterationCount = 0) => {
+    const transformed = await transformer(initialValue, iterationCount);
+    return (await predicate(transformed, iterationCount))
+        ? transformed
+        : until(predicate, transformer, transformed, iterationCount + 1);
+};
+
+const DEFAULT_BATCH_ITEMS_MAX_RETRIES = 50;
+let queue;
+const makeKey = (key) => {
+    if (typeof key === 'string') {
+        return { id: key };
+    }
+    return key;
+};
+const deleteItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ReturnValues = 'NONE' } = options;
+        const command = new libDynamodb.DeleteCommand({
+            Key: makeKey(key),
+            ReturnValues,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const getItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new libDynamodb.GetCommand({
+            ConsistentRead,
+            Key: makeKey(key),
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const putItem = async (config, documentClient, table, Item) => {
+    try {
+        const { expires, ...data } = Item;
+        const command = new libDynamodb.PutCommand({
+            Item: expires === null ? data : Item,
+            TableName: config.tableNamePrefix + table,
+        });
+        return (await queue.schedule(() => documentClient.send(command))) && Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const updateItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const command = new libDynamodb.UpdateCommand({
+            Key: makeKey(key),
+            ...options,
+            ReturnValues: 'ALL_NEW',
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(async () => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const hasNoUnprocessedItems = ({ UnprocessedItems, }) => !UnprocessedItems || Object.keys(UnprocessedItems).length === 0;
+const makeBatchWriteRequestWithExponentialBackoff = (documentClient, table, batchWriteMaxRetries) => async (batchItems, retries) => {
+    if (retries > batchWriteMaxRetries) {
+        throw new Error('Reached maximum number of DynamoDB BatchWrite request retries.');
+    }
+    const command = new libDynamodb.BatchWriteCommand({
+        RequestItems: batchItems.UnprocessedItems ?? {
+            [table]: batchItems.map((Item) => ({
+                PutRequest: { Item },
+            })),
+        },
+        ReturnConsumedCapacity: 'TOTAL',
+        ReturnItemCollectionMetrics: 'SIZE',
+    });
+    return documentClient.send(command);
+};
+const batchWriteItems = async (config, documentClient, table, items) => {
+    try {
+        const batches = splitEvery(25, items);
+        const result = await Promise.all(batches.map(async (batch) => queue.schedule(async () => until(hasNoUnprocessedItems, makeBatchWriteRequestWithExponentialBackoff(documentClient, config.tableNamePrefix + table, config.batchWriteMaxRetries ?? DEFAULT_BATCH_ITEMS_MAX_RETRIES), batch))));
+        return result && items;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const scanTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new libDynamodb.ScanCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const queryTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new libDynamodb.QueryCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const dynamodb = (userConfig) => {
+    const { clientConfig, ...config } = {
+        ...DEFAULT_AWS_CONFIGURATION,
+        ...userConfig,
+    };
+    const ddbClient = new clientDynamodb.DynamoDBClient(config.dockerConfig
+        ? {
+            credentials: {
+                accessKeyId: 'foobarkey',
+                secretAccessKey: 'foobarsecret',
+            },
+            endpoint: config.dockerConfig.endpoint,
+            region: 'local',
+            retryMode: 'standard',
+        }
+        : { retryMode: 'standard', ...clientConfig });
+    const documentClient = libDynamodb.DynamoDBDocumentClient.from(ddbClient);
+    queue = new Bottleneck({
+        maxConcurrent: config.queueConfig?.queueConcurrency ?? 25,
+        minTime: config.queueConfig?.queueDelay ?? undefined,
+    });
+    return {
+        batchWriteItems: (table, items) => batchWriteItems(config, documentClient, table, items),
+        deleteItem: (table, key, options) => deleteItem(config, documentClient, table, key, options),
+        getItem: (table, key, options) => getItem(config, documentClient, table, key, options),
+        putItem: (table, item) => putItem(config, documentClient, table, item),
+        queryTable: (table, options) => queryTable(config, documentClient, table, options),
+        scanTable: (table, options) => scanTable(config, documentClient, table, options),
+        updateItem: (table, key, options) => updateItem(config, documentClient, table, key, options),
+    };
+};
+
 const invokeLambda = async (lambdaClient, config, invocationType, payload) => {
     try {
         const invokeCommand = new clientLambda.InvokeCommand({
@@ -198,6 +384,7 @@ const logger = (config) => {
 
 exports.DEFAULT_PARAMETER_STORE_CLIENT_CONFIG = DEFAULT_CLIENT_CONFIG$1;
 exports.DEFAULT_SES_CLIENT_CONFIG = DEFAULT_CLIENT_CONFIG;
+exports.dynamodb = dynamodb;
 exports.lambda = lambda;
 exports.logger = logger;
 exports.parameterStore = parameterStore;

package/dist/lib.esm.js

@@ -1,3 +1,6 @@
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { DynamoDBDocumentClient, DeleteCommand, GetCommand, PutCommand, UpdateCommand, ScanCommand, QueryCommand, BatchWriteCommand } from '@aws-sdk/lib-dynamodb';
+import Bottleneck from 'bottleneck';
 import { LambdaClient, InvokeCommand } from '@aws-sdk/client-lambda';
 import { SSMClient, GetParametersCommand, GetParameterCommand } from '@aws-sdk/client-ssm';
 import * as aws from '@aws-sdk/client-ses';
@@ -8,6 +11,189 @@ const DEFAULT_AWS_CONFIGURATION = {
     region: 'eu-central-2',
 };
 
+const splitEvery = (every, list) => Array.from({ length: Math.ceil(list.length / every) }).map((_, count) => list.slice(count * every, count * every + every));
+const until = async (predicate, transformer, initialValue, iterationCount = 0) => {
+    const transformed = await transformer(initialValue, iterationCount);
+    return (await predicate(transformed, iterationCount))
+        ? transformed
+        : until(predicate, transformer, transformed, iterationCount + 1);
+};
+
+const DEFAULT_BATCH_ITEMS_MAX_RETRIES = 50;
+let queue;
+const makeKey = (key) => {
+    if (typeof key === 'string') {
+        return { id: key };
+    }
+    return key;
+};
+const deleteItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ReturnValues = 'NONE' } = options;
+        const command = new DeleteCommand({
+            Key: makeKey(key),
+            ReturnValues,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const getItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new GetCommand({
+            ConsistentRead,
+            Key: makeKey(key),
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const putItem = async (config, documentClient, table, Item) => {
+    try {
+        const { expires, ...data } = Item;
+        const command = new PutCommand({
+            Item: expires === null ? data : Item,
+            TableName: config.tableNamePrefix + table,
+        });
+        return (await queue.schedule(() => documentClient.send(command))) && Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const updateItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const command = new UpdateCommand({
+            Key: makeKey(key),
+            ...options,
+            ReturnValues: 'ALL_NEW',
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(async () => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const hasNoUnprocessedItems = ({ UnprocessedItems, }) => !UnprocessedItems || Object.keys(UnprocessedItems).length === 0;
+const makeBatchWriteRequestWithExponentialBackoff = (documentClient, table, batchWriteMaxRetries) => async (batchItems, retries) => {
+    if (retries > batchWriteMaxRetries) {
+        throw new Error('Reached maximum number of DynamoDB BatchWrite request retries.');
+    }
+    const command = new BatchWriteCommand({
+        RequestItems: batchItems.UnprocessedItems ?? {
+            [table]: batchItems.map((Item) => ({
+                PutRequest: { Item },
+            })),
+        },
+        ReturnConsumedCapacity: 'TOTAL',
+        ReturnItemCollectionMetrics: 'SIZE',
+    });
+    return documentClient.send(command);
+};
+const batchWriteItems = async (config, documentClient, table, items) => {
+    try {
+        const batches = splitEvery(25, items);
+        const result = await Promise.all(batches.map(async (batch) => queue.schedule(async () => until(hasNoUnprocessedItems, makeBatchWriteRequestWithExponentialBackoff(documentClient, config.tableNamePrefix + table, config.batchWriteMaxRetries ?? DEFAULT_BATCH_ITEMS_MAX_RETRIES), batch))));
+        return result && items;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const scanTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new ScanCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const queryTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new QueryCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const dynamodb = (userConfig) => {
+    const { clientConfig, ...config } = {
+        ...DEFAULT_AWS_CONFIGURATION,
+        ...userConfig,
+    };
+    const ddbClient = new DynamoDBClient(config.dockerConfig
+        ? {
+            credentials: {
+                accessKeyId: 'foobarkey',
+                secretAccessKey: 'foobarsecret',
+            },
+            endpoint: config.dockerConfig.endpoint,
+            region: 'local',
+            retryMode: 'standard',
+        }
+        : { retryMode: 'standard', ...clientConfig });
+    const documentClient = DynamoDBDocumentClient.from(ddbClient);
+    queue = new Bottleneck({
+        maxConcurrent: config.queueConfig?.queueConcurrency ?? 25,
+        minTime: config.queueConfig?.queueDelay ?? undefined,
+    });
+    return {
+        batchWriteItems: (table, items) => batchWriteItems(config, documentClient, table, items),
+        deleteItem: (table, key, options) => deleteItem(config, documentClient, table, key, options),
+        getItem: (table, key, options) => getItem(config, documentClient, table, key, options),
+        putItem: (table, item) => putItem(config, documentClient, table, item),
+        queryTable: (table, options) => queryTable(config, documentClient, table, options),
+        scanTable: (table, options) => scanTable(config, documentClient, table, options),
+        updateItem: (table, key, options) => updateItem(config, documentClient, table, key, options),
+    };
+};
+
 const invokeLambda = async (lambdaClient, config, invocationType, payload) => {
     try {
         const invokeCommand = new InvokeCommand({
@@ -175,4 +361,4 @@ const logger = (config) => {
     });
 };
 
-export { DEFAULT_CLIENT_CONFIG$1 as DEFAULT_PARAMETER_STORE_CLIENT_CONFIG, DEFAULT_CLIENT_CONFIG as DEFAULT_SES_CLIENT_CONFIG, lambda, logger, parameterStore, ses };
+export { DEFAULT_CLIENT_CONFIG$1 as DEFAULT_PARAMETER_STORE_CLIENT_CONFIG, DEFAULT_CLIENT_CONFIG as DEFAULT_SES_CLIENT_CONFIG, dynamodb, lambda, logger, parameterStore, ses };