@allthings/cloud-toolkit 1.3.0-dev.2 → 1.4.0

package/README.md

@@ -3,17 +3,31 @@ Standardizes the setup of aws, datadog and other things
 
 ## Available functions
 ### parameter store
-| function   | description                                                                    |
-|------------|--------------------------------------------------------------------------------|
-| getSecret  | Retrieves a single secret from parameter store by providing the path           |
-| getSecrets | Retrieves multiple secrets from parameter store by providing an array of paths |
+| function   | parameters                              | description                                                                    |
+|------------|-----------------------------------------|--------------------------------------------------------------------------------|
+| getSecret  | parameter<br/>defaultValue (optional)   | Retrieves a single secret from parameter store by providing the path           |
+| getSecrets | parameters<br/>defaultValues (optional) | Retrieves multiple secrets from parameter store by providing an array of paths |
 
 ### ses
-| function | description                                    |
-|----------|------------------------------------------------|
-| sendMail | Sends an email via SES or provided SMTP config |
+| function | parameters | description                                    |
+|----------|------------|------------------------------------------------|
+| sendMail | options    | Sends an email via SES or provided SMTP config |
 
 ### lambda
-| function     | description                                          |
-|--------------|------------------------------------------------------|
-| invokeLambda | Invokes set lambda with invocations type and payload |
+| function     | parameters                 | description                                          |
+|--------------|----------------------------|------------------------------------------------------|
+| invokeLambda | invocationType<br/>payload | Invokes set lambda with invocations type and payload |
+
+### dynamodb
+| function        | parameters                           | description                                              |
+|-----------------|--------------------------------------|----------------------------------------------------------|
+| batchWriteItems | table<br/>items                      | Creates multiple item in a defined table                 |
+| deleteItem      | table<br/>key<br/>options (optional) | Deletes database item from defined table by given key    |
+| getItem         | table<br/>key<br/>options (optional) | Returns item from Database by defined table and key      |
+| putItem         | table<br/>key                        | Creates one item in defined table                        |
+| queryTable      | table<br/>options (optional)         | Returns result items of query                            |
+| scanTable       | table<br/>options (optional)         | Returns result items of scan                             |
+| updateItem      | table<br/>key<br/>options (optional) | Updates item in database by provided table and given key |
+
+### logger
+A winston instance will be created with default log levels https://github.com/winstonjs/winston?tab=readme-ov-file#logging

package/dist/lib.cjs.js

@@ -1,5 +1,8 @@
 'use strict';
 
+var clientDynamodb = require('@aws-sdk/client-dynamodb');
+var libDynamodb = require('@aws-sdk/lib-dynamodb');
+var Bottleneck = require('bottleneck');
 var clientLambda = require('@aws-sdk/client-lambda');
 var clientSsm = require('@aws-sdk/client-ssm');
 var aws = require('@aws-sdk/client-ses');
@@ -29,6 +32,189 @@ const DEFAULT_AWS_CONFIGURATION = {
     region: 'eu-central-2',
 };
 
+const splitEvery = (every, list) => Array.from({ length: Math.ceil(list.length / every) }).map((_, count) => list.slice(count * every, count * every + every));
+const until = async (predicate, transformer, initialValue, iterationCount = 0) => {
+    const transformed = await transformer(initialValue, iterationCount);
+    return (await predicate(transformed, iterationCount))
+        ? transformed
+        : until(predicate, transformer, transformed, iterationCount + 1);
+};
+
+const DEFAULT_BATCH_ITEMS_MAX_RETRIES = 50;
+let queue;
+const makeKey = (key) => {
+    if (typeof key === 'string') {
+        return { id: key };
+    }
+    return key;
+};
+const deleteItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ReturnValues = 'NONE' } = options;
+        const command = new libDynamodb.DeleteCommand({
+            Key: makeKey(key),
+            ReturnValues,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const getItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new libDynamodb.GetCommand({
+            ConsistentRead,
+            Key: makeKey(key),
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const putItem = async (config, documentClient, table, Item) => {
+    try {
+        const { expires, ...data } = Item;
+        const command = new libDynamodb.PutCommand({
+            Item: expires === null ? data : Item,
+            TableName: config.tableNamePrefix + table,
+        });
+        return (await queue.schedule(() => documentClient.send(command))) && Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const updateItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const command = new libDynamodb.UpdateCommand({
+            Key: makeKey(key),
+            ...options,
+            ReturnValues: 'ALL_NEW',
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(async () => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const hasNoUnprocessedItems = ({ UnprocessedItems, }) => !UnprocessedItems || Object.keys(UnprocessedItems).length === 0;
+const makeBatchWriteRequestWithExponentialBackoff = (documentClient, table, batchWriteMaxRetries) => async (batchItems, retries) => {
+    if (retries > batchWriteMaxRetries) {
+        throw new Error('Reached maximum number of DynamoDB BatchWrite request retries.');
+    }
+    const command = new libDynamodb.BatchWriteCommand({
+        RequestItems: batchItems.UnprocessedItems ?? {
+            [table]: batchItems.map((Item) => ({
+                PutRequest: { Item },
+            })),
+        },
+        ReturnConsumedCapacity: 'TOTAL',
+        ReturnItemCollectionMetrics: 'SIZE',
+    });
+    return documentClient.send(command);
+};
+const batchWriteItems = async (config, documentClient, table, items) => {
+    try {
+        const batches = splitEvery(25, items);
+        const result = await Promise.all(batches.map(async (batch) => queue.schedule(async () => until(hasNoUnprocessedItems, makeBatchWriteRequestWithExponentialBackoff(documentClient, config.tableNamePrefix + table, config.batchWriteMaxRetries ?? DEFAULT_BATCH_ITEMS_MAX_RETRIES), batch))));
+        return result && items;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const scanTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new libDynamodb.ScanCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const queryTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new libDynamodb.QueryCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const dynamodb = (userConfig) => {
+    const { clientConfig, ...config } = {
+        ...DEFAULT_AWS_CONFIGURATION,
+        ...userConfig,
+    };
+    const ddbClient = new clientDynamodb.DynamoDBClient(config.dockerConfig
+        ? {
+            credentials: {
+                accessKeyId: 'foobarkey',
+                secretAccessKey: 'foobarsecret',
+            },
+            endpoint: config.dockerConfig.endpoint,
+            region: 'local',
+            retryMode: 'standard',
+        }
+        : { retryMode: 'standard', ...clientConfig });
+    const documentClient = libDynamodb.DynamoDBDocumentClient.from(ddbClient);
+    queue = new Bottleneck({
+        maxConcurrent: config.queueConfig?.queueConcurrency ?? 25,
+        minTime: config.queueConfig?.queueDelay ?? undefined,
+    });
+    return {
+        batchWriteItems: (table, items) => batchWriteItems(config, documentClient, table, items),
+        deleteItem: (table, key, options) => deleteItem(config, documentClient, table, key, options),
+        getItem: (table, key, options) => getItem(config, documentClient, table, key, options),
+        putItem: (table, item) => putItem(config, documentClient, table, item),
+        queryTable: (table, options) => queryTable(config, documentClient, table, options),
+        scanTable: (table, options) => scanTable(config, documentClient, table, options),
+        updateItem: (table, key, options) => updateItem(config, documentClient, table, key, options),
+    };
+};
+
 const invokeLambda = async (lambdaClient, config, invocationType, payload) => {
     try {
         const invokeCommand = new clientLambda.InvokeCommand({
@@ -65,9 +251,9 @@ const DEFAULT_CLIENT_CONFIG$1 = {
     onlyDefault: false,
     prefix: process.env.SSM_PARAM_PATH,
 };
-const getSecrets = async (ssm, config, parameters_, defaultValue) => {
+const getSecrets = async (ssm, config, parameters_, defaultValues) => {
     if (config.onlyDefault) {
-        return [defaultValue];
+        return defaultValues;
     }
     try {
         const command = new clientSsm.GetParametersCommand({
@@ -75,13 +261,21 @@ const getSecrets = async (ssm, config, parameters_, defaultValue) => {
             WithDecryption: true,
         });
         const parameters = await ssm.send(command);
-        return (parameters?.Parameters?.map((parameter) => parameter.Value ?? defaultValue) ?? [defaultValue]);
+        return (parameters?.Parameters?.map((parameter) => {
+            const defaultIfNoParameter = !parameter.Value && parameter.Name
+                ? defaultValues.find((defaultValue) => defaultValue.name === parameter.Name)
+                : defaultValues[0];
+            return {
+                name: parameter.Name ?? defaultIfNoParameter?.name ?? '',
+                value: parameter.Value ?? defaultIfNoParameter?.value ?? '',
+            };
+        }) ?? defaultValues);
     }
     catch (error) {
         if (config.logger) {
             config.logger('error', error);
         }
-        return [defaultValue];
+        return defaultValues;
     }
 };
 const getSecret = async (ssm, config, parameter_, defaultValue) => {
@@ -110,7 +304,7 @@ const parameterStore = (userConfig) => {
     const ssm = new clientSsm.SSMClient(clientConfig);
     return {
         getSecret: (parameter, defaultValue = '') => getSecret(ssm, config, parameter, defaultValue),
-        getSecrets: (parameters, defaultValue = '') => getSecrets(ssm, config, parameters, defaultValue),
+        getSecrets: (parameters, defaultValues = []) => getSecrets(ssm, config, parameters, defaultValues),
     };
 };
 
@@ -184,12 +378,13 @@ const logger = (config) => {
     }
     return winston.createLogger({
         ...loggerConfig,
-        transports: [new winston.transports.Console()],
+        transports: [new winston.transports.Console({ forceConsole: true })],
     });
 };
 
 exports.DEFAULT_PARAMETER_STORE_CLIENT_CONFIG = DEFAULT_CLIENT_CONFIG$1;
 exports.DEFAULT_SES_CLIENT_CONFIG = DEFAULT_CLIENT_CONFIG;
+exports.dynamodb = dynamodb;
 exports.lambda = lambda;
 exports.logger = logger;
 exports.parameterStore = parameterStore;

package/dist/lib.esm.js

@@ -1,3 +1,6 @@
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { DynamoDBDocumentClient, DeleteCommand, GetCommand, PutCommand, UpdateCommand, ScanCommand, QueryCommand, BatchWriteCommand } from '@aws-sdk/lib-dynamodb';
+import Bottleneck from 'bottleneck';
 import { LambdaClient, InvokeCommand } from '@aws-sdk/client-lambda';
 import { SSMClient, GetParametersCommand, GetParameterCommand } from '@aws-sdk/client-ssm';
 import * as aws from '@aws-sdk/client-ses';
@@ -8,6 +11,189 @@ const DEFAULT_AWS_CONFIGURATION = {
     region: 'eu-central-2',
 };
 
+const splitEvery = (every, list) => Array.from({ length: Math.ceil(list.length / every) }).map((_, count) => list.slice(count * every, count * every + every));
+const until = async (predicate, transformer, initialValue, iterationCount = 0) => {
+    const transformed = await transformer(initialValue, iterationCount);
+    return (await predicate(transformed, iterationCount))
+        ? transformed
+        : until(predicate, transformer, transformed, iterationCount + 1);
+};
+
+const DEFAULT_BATCH_ITEMS_MAX_RETRIES = 50;
+let queue;
+const makeKey = (key) => {
+    if (typeof key === 'string') {
+        return { id: key };
+    }
+    return key;
+};
+const deleteItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ReturnValues = 'NONE' } = options;
+        const command = new DeleteCommand({
+            Key: makeKey(key),
+            ReturnValues,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const getItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new GetCommand({
+            ConsistentRead,
+            Key: makeKey(key),
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result.Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const putItem = async (config, documentClient, table, Item) => {
+    try {
+        const { expires, ...data } = Item;
+        const command = new PutCommand({
+            Item: expires === null ? data : Item,
+            TableName: config.tableNamePrefix + table,
+        });
+        return (await queue.schedule(() => documentClient.send(command))) && Item;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const updateItem = async (config, documentClient, table, key, options = {}) => {
+    try {
+        const command = new UpdateCommand({
+            Key: makeKey(key),
+            ...options,
+            ReturnValues: 'ALL_NEW',
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(async () => documentClient.send(command));
+        return result.Attributes;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const hasNoUnprocessedItems = ({ UnprocessedItems, }) => !UnprocessedItems || Object.keys(UnprocessedItems).length === 0;
+const makeBatchWriteRequestWithExponentialBackoff = (documentClient, table, batchWriteMaxRetries) => async (batchItems, retries) => {
+    if (retries > batchWriteMaxRetries) {
+        throw new Error('Reached maximum number of DynamoDB BatchWrite request retries.');
+    }
+    const command = new BatchWriteCommand({
+        RequestItems: batchItems.UnprocessedItems ?? {
+            [table]: batchItems.map((Item) => ({
+                PutRequest: { Item },
+            })),
+        },
+        ReturnConsumedCapacity: 'TOTAL',
+        ReturnItemCollectionMetrics: 'SIZE',
+    });
+    return documentClient.send(command);
+};
+const batchWriteItems = async (config, documentClient, table, items) => {
+    try {
+        const batches = splitEvery(25, items);
+        const result = await Promise.all(batches.map(async (batch) => queue.schedule(async () => until(hasNoUnprocessedItems, makeBatchWriteRequestWithExponentialBackoff(documentClient, config.tableNamePrefix + table, config.batchWriteMaxRetries ?? DEFAULT_BATCH_ITEMS_MAX_RETRIES), batch))));
+        return result && items;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const scanTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new ScanCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const queryTable = async (config, documentClient, table, options = {}) => {
+    try {
+        const { ConsistentRead = false } = options;
+        const command = new QueryCommand({
+            ...options,
+            ConsistentRead,
+            TableName: config.tableNamePrefix + table,
+        });
+        const result = await queue.schedule(() => documentClient.send(command));
+        return result;
+    }
+    catch (error) {
+        if (config.logger) {
+            config.logger('error', error);
+        }
+        return undefined;
+    }
+};
+const dynamodb = (userConfig) => {
+    const { clientConfig, ...config } = {
+        ...DEFAULT_AWS_CONFIGURATION,
+        ...userConfig,
+    };
+    const ddbClient = new DynamoDBClient(config.dockerConfig
+        ? {
+            credentials: {
+                accessKeyId: 'foobarkey',
+                secretAccessKey: 'foobarsecret',
+            },
+            endpoint: config.dockerConfig.endpoint,
+            region: 'local',
+            retryMode: 'standard',
+        }
+        : { retryMode: 'standard', ...clientConfig });
+    const documentClient = DynamoDBDocumentClient.from(ddbClient);
+    queue = new Bottleneck({
+        maxConcurrent: config.queueConfig?.queueConcurrency ?? 25,
+        minTime: config.queueConfig?.queueDelay ?? undefined,
+    });
+    return {
+        batchWriteItems: (table, items) => batchWriteItems(config, documentClient, table, items),
+        deleteItem: (table, key, options) => deleteItem(config, documentClient, table, key, options),
+        getItem: (table, key, options) => getItem(config, documentClient, table, key, options),
+        putItem: (table, item) => putItem(config, documentClient, table, item),
+        queryTable: (table, options) => queryTable(config, documentClient, table, options),
+        scanTable: (table, options) => scanTable(config, documentClient, table, options),
+        updateItem: (table, key, options) => updateItem(config, documentClient, table, key, options),
+    };
+};
+
 const invokeLambda = async (lambdaClient, config, invocationType, payload) => {
     try {
         const invokeCommand = new InvokeCommand({
@@ -44,9 +230,9 @@ const DEFAULT_CLIENT_CONFIG$1 = {
     onlyDefault: false,
     prefix: process.env.SSM_PARAM_PATH,
 };
-const getSecrets = async (ssm, config, parameters_, defaultValue) => {
+const getSecrets = async (ssm, config, parameters_, defaultValues) => {
     if (config.onlyDefault) {
-        return [defaultValue];
+        return defaultValues;
     }
     try {
         const command = new GetParametersCommand({
@@ -54,13 +240,21 @@ const getSecrets = async (ssm, config, parameters_, defaultValue) => {
             WithDecryption: true,
         });
         const parameters = await ssm.send(command);
-        return (parameters?.Parameters?.map((parameter) => parameter.Value ?? defaultValue) ?? [defaultValue]);
+        return (parameters?.Parameters?.map((parameter) => {
+            const defaultIfNoParameter = !parameter.Value && parameter.Name
+                ? defaultValues.find((defaultValue) => defaultValue.name === parameter.Name)
+                : defaultValues[0];
+            return {
+                name: parameter.Name ?? defaultIfNoParameter?.name ?? '',
+                value: parameter.Value ?? defaultIfNoParameter?.value ?? '',
+            };
+        }) ?? defaultValues);
     }
     catch (error) {
         if (config.logger) {
             config.logger('error', error);
         }
-        return [defaultValue];
+        return defaultValues;
     }
 };
 const getSecret = async (ssm, config, parameter_, defaultValue) => {
@@ -89,7 +283,7 @@ const parameterStore = (userConfig) => {
     const ssm = new SSMClient(clientConfig);
     return {
         getSecret: (parameter, defaultValue = '') => getSecret(ssm, config, parameter, defaultValue),
-        getSecrets: (parameters, defaultValue = '') => getSecrets(ssm, config, parameters, defaultValue),
+        getSecrets: (parameters, defaultValues = []) => getSecrets(ssm, config, parameters, defaultValues),
     };
 };
 
@@ -163,8 +357,8 @@ const logger = (config) => {
     }
     return createLogger({
         ...loggerConfig,
-        transports: [new transports.Console()],
+        transports: [new transports.Console({ forceConsole: true })],
     });
 };
 
-export { DEFAULT_CLIENT_CONFIG$1 as DEFAULT_PARAMETER_STORE_CLIENT_CONFIG, DEFAULT_CLIENT_CONFIG as DEFAULT_SES_CLIENT_CONFIG, lambda, logger, parameterStore, ses };
+export { DEFAULT_CLIENT_CONFIG$1 as DEFAULT_PARAMETER_STORE_CLIENT_CONFIG, DEFAULT_CLIENT_CONFIG as DEFAULT_SES_CLIENT_CONFIG, dynamodb, lambda, logger, parameterStore, ses };