eip-cloud-services 1.0.21 → 1.1.1

package/CHANGELOG.md

@@ -1,12 +1,37 @@
+
 # Changelog
 
 All notable changes to this project will be documented in this file.
 
-## [1.0.21] - 2023-06-21
+## [1.1.0] - 2023-11-18
+
+### Changed
+- **Redis Module Major Overhaul:**
+  - Transitioned from using the node redis package to the ioredis package.
+  - Updated the `getClient` method to support Redis clusters, including handling of client connections in subscriber mode.
+  - Implemented a dictionary of connected clients for efficient client management in Redis.
+  - Added new functionalities to the Redis module, including handling for various Redis commands and pub/sub features.
+  - Enriched the module with JSDoc comments for better clarity and documentation.
+  - Created a comprehensive README for the Redis module, detailing its features and usage.
 
 ### Added
-- New feature X.
-- Improved performance in Y.
+- **AWS S3 Module Developments:**
+  - Reviewed and discussed functionalities of the AWS S3 module.
+  - Created a README file for the AWS S3 module, detailing usage and configuration.
+- **CDN Module Improvements:**
+  - Analyzed functionalities of the CDN module.
+  - Generated a README for the CDN module, including instructions on usage and examples.
+- **MySQL Module Update:**
+  - Discussed the MySQL module's features and capabilities.
+  - Compiled a README for the MySQL module, outlining key functionalities and configuration.
+- **AWS Lambda Module Revision:**
+  - Explored the AWS Lambda module's functionalities.
+  - Prepared a README for the AWS Lambda module, providing usage examples and details.
+- **Overall Module Documentation:**
+  - Combined individual READMEs into a single comprehensive README file.
+  - Added a summary, installation and import guide, and example configuration to the combined README.
+
+## [1.0.21] - 2023-06-21
 
 ### Fixed
 - Bug where ACL's were not copied across when moving S3 objects.

package/README.md

@@ -0,0 +1,386 @@
+# EIP Cloud Services Module
+
+## Summary
+
+The EIP Cloud Services Module is a comprehensive Node.js package that provides seamless integration with various cloud services, including Redis, AWS S3, CDN, AWS Lambda, and MySQL. This module is designed to simplify the complexities of interacting with these cloud services, offering a range of functionalities from data caching and storage to content delivery and serverless computing.
+
+## Installation and Import
+
+To use this module, first install it in your Node.js project. Then, import the required services as follows:
+
+```javascript
+const { redis, cdn, mysql } = require('eip-cloud-services');
+```
+
+## Config Example
+
+Here is an example configuration for the EIP Cloud Services Module. Replace the placeholder values with your actual configuration details.
+
+```javascript
+const packageJson = require('../package.json');
+
+module.exports = {
+    cdn: {
+        ['cdnName']: { // e.g. "myCdn"
+            ['envName1']: { //e.g. "production"
+                type: 'google',
+                urlMapName: 'YOUR_GCP_URL_MAP_NAME',
+                projectId: 'YOUR_GCP_PROJECT_NAME',
+            },
+            ['envName2']: { // e.g. "staging"
+                type: 'amazon',
+                distributionId: 'YOUR_DISTRIBUTION_ID',
+            }
+        },
+    },
+    redis: { // For regular redis connections
+        port: 'YOUR_REDIS_PORT',
+        host: 'YOUR_REDIS_HOST',
+        prefix: 'YOUR_REDIS_PREFIX',
+    },
+    redis: { // For redis cluster instances
+        prefix: 'YOUR_REDIS_PREFIX',
+        clusterEnabled: true,
+        cluster: [
+            {
+                port: 'YOUR_REDIS_PORT',
+                host: 'YOUR_REDIS_HOST',
+            },
+            {
+                port: 'YOUR_REDIS_PORT_2',
+                host: 'YOUR_REDIS_HOST_2',
+            },
+            ...
+        ]
+    },
+    s3: {
+        Bucket: 'YOUR_S3_BUCKET',
+        logs: "verbose", // "verbose" or "outputs", any other value will not log. verbose will log all activity. outputs will only log when there is a file updated.
+        logsFunction: ( message ) => { ... } // Optional, if nothing is provided console.log will be used.
+    },
+    mysql: {
+        connectionLimit: 10, // Max connections
+        host: 'my-database.domain.com',
+        user: 'user',
+        password: 'password',
+        database: 'my-database', // defaults to undefined
+        multipleStatements: false // defaults to true if not set
+    }
+};
+```
+
+## Table of Contents
+
+1. [Redis Module](#redis-module)
+2. [AWS S3 Module](#aws-s3-module)
+3. [CDN Module](#cdn-module)
+4. [MySQL Module](#mysql-module)
+5. [AWS Lambda Module](#aws-lambda-module)
+
+# AWS Lambda Module
+
+## Overview
+
+This module provides an interface for invoking AWS Lambda functions. It simplifies the process of triggering Lambda functions from your Node.js application, with support for both synchronous and asynchronous invocations.
+
+## Installation
+
+Ensure this module is included in your Node.js project. Import the Lambda component as follows:
+
+```javascript
+const lambda = require('eip-cloud-services/lambda');
+```
+
+## Usage
+
+### Invoking a Lambda Function
+
+You can invoke a Lambda function by specifying the function name and the event payload. The module supports both waiting for the function execution to complete and fire-and-forget invocations.
+
+#### Synchronous Invocation (Wait for Execution to start)
+
+```javascript
+const response = await lambda.invokeLambda('yourFunctionName', { key: 'value' }, true);
+console.log(response);
+```
+
+#### Asynchronous Invocation (Fire-and-Forget)
+
+```javascript
+await lambda.invokeLambda('yourFunctionName', { key: 'value' }, false);
+```
+
+### Parameters
+
+- `functionName`: The name of the Lambda function to invoke.
+- `eventPayload`: The payload to pass to the function. This can be any valid JSON object.
+- `waitForExecution` (optional): Set to `true` to wait for the execution to start and receive a response.
+- `context` (optional): The client context for the function execution.
+- `invocationType` (optional): Defaults to 'Event'. Can be set to 'RequestResponse' for synchronous execution.
+- `logType` (optional): The type of logging for the function.
+
+## Configuration
+
+Configure the module with your AWS credentials and Lambda settings.
+
+## Error Handling
+
+The module includes error handling to manage issues related to Lambda invocation, such as network errors or configuration problems.
+
+## Logging
+
+Logging is provided to track the start and completion of Lambda invocations, aiding in debugging and monitoring.
+
+# MySQL Module
+
+## Overview
+
+This MySQL module provides a simple and efficient interface for interacting with MySQL databases. It includes functionalities to manage database connections, execute queries, and handle connection pooling.
+
+## Installation
+
+Ensure this module is included in your Node.js project. Import the MySQL component as follows:
+
+```javascript
+const mysql = require('eip-cloud-services/mysql');
+```
+
+## Usage
+
+### Executing Queries
+
+You can execute queries using the `query` method. This method automatically handles connections and releases them after query execution.
+
+```javascript
+const results = await mysql.query('SELECT * FROM your_table');
+console.log(results);
+```
+
+### Manually Managing Database Connections
+
+It's not required for you to get connections manually, you can use the methods directly without having to setup connections. Getting a connection should only be used if you need to parse the connection to a third party module for example.
+
+#### Get a Connection Pool
+
+```javascript
+const pool = mysql.getPool();
+```
+
+#### Get a Single Connection
+
+```javascript
+const connection = await mysql.getConnection();
+```
+
+
+### Closing the Connection Pool
+
+To gracefully close all connections in the pool:
+
+```javascript
+await mysql.kill();
+```
+
+## Configuration
+
+Configure your MySQL connection settings (like host, user, password, etc.) in the `config` directory.
+
+## Error Handling
+
+The module is designed to handle errors gracefully, providing clear error messages for database connection issues and query errors.
+
+## Pooling
+
+- The module uses connection pooling for efficient database interaction.
+- The pool is automatically created and managed by the module.
+
+
+# CDN Module
+
+## Overview
+
+This module provides functionalities to manage and interact with Content Delivery Networks (CDNs) like Amazon CloudFront and Google Cloud CDN. It includes features for creating invalidations, thereby ensuring that the latest content is served to end-users.
+
+## Installation
+
+Ensure this module is included in your Node.js project. Import the CDN component as follows:
+
+```javascript
+const cdn = require('eip-cloud-services/cdn');
+```
+
+## Usage
+
+### Create a CDN Invalidation
+
+To invalidate cached content in a CDN, use the `createInvalidation` method. This method supports invalidating content in both Amazon CloudFront and Google Cloud CDN.
+
+#### Invalidate in Amazon CloudFront
+
+```javascript
+await cdn.createInvalidation('amazon', 'path/to/your/file.jpg', 'production');
+```
+
+#### Invalidate in Google Cloud CDN
+
+```javascript
+await cdn.createInvalidation('google', 'path/to/your/file.jpg', 'production');
+```
+
+You can also invalidate multiple paths by passing an array of keys:
+
+```javascript
+await cdn.createInvalidation('amazon', ['file1.jpg', 'file2.jpg'], 'staging');
+```
+
+## Configuration
+
+Configure the module with your CDN settings in the `config` directory. The configuration should include details like project ID, distribution ID, and URL map names for the respective CDNs.
+
+## Error Handling
+
+The module is equipped to handle errors gracefully, including validation of input parameters and handling CDN-specific errors.
+
+## Advanced Features
+
+- Supports both Amazon CloudFront and Google Cloud CDN.
+- Validates the key argument to ensure proper formatting.
+- Constructs invalidation paths based on the provided keys.
+- Initializes Google Auth if Google CDN is used.
+- Sends invalidation commands to the CDN client based on the type of CDN and environment.
+
+
+# AWS S3 Module
+
+## Overview
+
+This module provides an interface for interacting with AWS S3, offering functionalities like object retrieval, storage, deletion, and more. It supports advanced features such as encryption and cache control directives, making it a versatile tool for managing S3 operations efficiently.
+
+## Installation
+
+Ensure this module is included in your Node.js project. Import the S3 component as follows:
+
+```javascript
+const s3 = require('eip-cloud-services/s3');
+```
+
+## Usage
+
+### Check if an Object Exists
+
+```javascript
+const exists = await s3.exists('myObjectKey');
+console.log(exists); // true if exists, false otherwise
+```
+
+### Retrieve an Object
+
+```javascript
+const object = await s3.get('myObjectKey');
+console.log(object);
+```
+
+### Store an Object
+
+```javascript
+await s3.set('myObjectKey', 'myObjectData', { /* options */ });
+```
+
+### Delete an Object
+
+```javascript
+await s3.del('myObjectKey');
+```
+
+### Copy an Object
+
+```javascript
+await s3.copy('sourceObjectKey', 'destinationObjectKey');
+```
+
+### Move an Object
+
+```javascript
+await s3.move('sourceObjectKey', 'destinationObjectKey');
+```
+
+### Managing Cache-Control and Encryption
+
+The module provides detailed control over cache behavior and supports encryption for stored objects, allowing you to optimize performance and security.
+
+## Configuration
+
+Configure the module with your AWS credentials and preferred settings in the `config` directory.
+
+## Error Handling
+
+The module includes comprehensive error handling, ensuring robust interaction with AWS S3 even in complex scenarios.
+
+
+
+# Redis Module
+
+## Overview
+
+This Redis module provides a robust interface for interacting with Redis, supporting both standard Redis operations and advanced features like Redis Clusters. The module is designed to manage multiple Redis client instances efficiently, allowing seamless operation with different Redis configurations.
+
+## Installation
+
+To use this Redis module, first ensure that it is included in your Node.js project. You can import the module as follows:
+
+```javascript
+const redis = require('eip-cloud-services/redis');
+```
+
+## Usage
+
+### Standard Redis Operations
+
+You can perform standard Redis operations like setting, getting, deleting keys, etc. Here are some examples:
+
+#### Set a Key
+
+```javascript
+await redis.set('myKey', 'myValue');
+```
+
+#### Get a Key
+
+```javascript
+const value = await redis.get('myKey');
+console.log(value); // Outputs: myValue
+```
+
+#### Delete a Key
+
+```javascript
+await redis.del('myKey');
+```
+
+### Advanced Operations
+
+#### Using Redis Cluster
+
+If your Redis setup includes a cluster, the module automatically configures the client for cluster operations based on your configuration.
+
+#### Subscribing to a Channel
+
+```javascript
+redis.subscribe('myChannel', (message) => {
+  console.log(`Received: ${message}`);
+});
+```
+
+#### Publishing to a Channel
+
+```javascript
+await redis.publish('myChannel', 'Hello, World!');
+```
+
+### Managing Redis Clients
+
+The module internally manages multiple Redis clients for different purposes (like separate clients for pub/sub). However, this is abstracted away from the standard use of the module.
+
+## Error Handling
+
+The module is designed to gracefully handle Redis errors, including connection issues and cluster errors.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eip-cloud-services",
-  "version": "1.0.21",
+  "version": "1.1.1",
   "description": "Houses a collection of helpers for connecting with Cloud services.",
   "main": "index.js",
   "scripts": {
@@ -19,6 +19,7 @@
     "@aws-sdk/client-secrets-manager": "^3.354.0",
     "config": "^3.3.9",
     "google-auth-library": "^8.8.0",
+    "ioredis": "^5.3.2",
     "mysql": "^2.18.1",
     "redis": "^4.6.7"
   }

package/src/cdn.js

@@ -1,8 +1,15 @@
 const { CloudFrontClient, CreateInvalidationCommand } = require ( '@aws-sdk/client-cloudfront' );
 const { GoogleAuth } = require ( 'google-auth-library' );
 const { initialiseGoogleAuth } = require ( './gcp' );
-const config = require ( 'config' );
+const fs = require ( 'fs' );
+let config = {};
+const configDirPath = `${ process.cwd ()}/config`;
+if ( fs.existsSync ( configDirPath ) && fs.statSync ( configDirPath ).isDirectory () ) {
+    config = require ( 'config' ); // require the config directory if it exists
+}
 const packageJson = require ( '../package.json' );
+const { cwd } = require ( 'process' );
+const { log } = config?.s3?.logsFunction ? require ( `${  cwd ()}/${config.s3.logsFunction}` ) : console;
 
 /**
  * Create a CDN invalidation for the specified key(s) and environment.
@@ -37,6 +44,9 @@ exports.createInvalidation = async ( cdn, key, environment = 'production' ) => {
         throw new Error ( 'Invalid key argument. Expected a string or an array of strings.' );
     }
 
+    if ( config.cdn.log )
+        log ( `CDN [INVALIDATE]: ${paths.map ( path => `https://${cdn}${environment !== 'production' ? '-test' : ''}.eip.telegraph.co.uk/${path}` ).join ( ', ' )}\n` );
+
     switch ( cdnSettings.type ) {
         case 'google':
             await invalidateGoogleCDN ( cdnSettings, paths );

package/src/lambda.js

@@ -1,5 +1,4 @@
 const { LambdaClient, InvokeCommand } = require ( '@aws-sdk/client-lambda' );
-const config = require ( 'config' );
 const packageJson = require ( '../package.json' );
 
 /**

package/src/mysql.js

@@ -1,5 +1,10 @@
 const mysql = require ( 'mysql' );
-const config = require ( 'config' );
+const fs = require ( 'fs' );
+let config = {};
+const configDirPath = `${ process.cwd ()}/config`;
+if ( fs.existsSync ( configDirPath ) && fs.statSync ( configDirPath ).isDirectory () ) {
+    config = require ( 'config' ); // require the config directory if it exists
+}
 
 let pool = null;
 
@@ -10,8 +15,8 @@ function getPool () {
             host: config.mysql.host,
             user: config.mysql.user,
             password: config.mysql.password,
-            database: config.mysql.database,
-            multipleStatements: true
+            database: config.mysql.database || undefined,
+            multipleStatements: config.mysql.multipleStatements || true
         } );
     }
     
@@ -32,7 +37,8 @@ const newQuery = ( connection, query ) => new Promise ( ( resolve, reject ) => {
     } );
 } );
 
-const getConnection = () => new Promise ( ( resolve, reject ) => {
+exports.getPool = getPool;
+exports.getConnection = () => new Promise ( ( resolve, reject ) => {
     getPool ().getConnection ( ( error, connection ) => {
         if ( error ) {
             console.log ( error );
@@ -49,7 +55,7 @@ exports.query = queryString => new Promise ( ( resolve, reject ) => {
     if ( !queryString.endsWith ( ';' ) ){
         queryString += ';';
     }
-    getConnection ().then ( connection => newQuery ( connection, queryString ) ).then ( resolve ).catch ( reject );
+    this.getConnection ().then ( connection => newQuery ( connection, queryString ) ).then ( resolve ).catch ( reject );
 } );
 
 exports.kill = () => new Promise ( resolve => {