eip-cloud-services 1.2.6 → 1.4.0

package/README.md

@@ -81,6 +81,51 @@ module.exports = {
 4. [MySQL Module](#mysql-module)
 5. [AWS Lambda Module](#aws-lambda-module)
 6. [AWS SQS Module](#aws-sqs-module)
+7. [AWS Secrets Manager Module](#aws-secrets-manager-module)
+
+# AWS Secrets Manager Module
+
+## Overview
+
+This module provides helpers for reading and creating AWS Secrets Manager secrets. Secret reads use a local tmp-directory cache by default to avoid unnecessary AWS requests. Use `bypassCache` when the caller must force a fresh AWS lookup.
+
+## Usage
+
+```javascript
+const { secrets } = require('eip-cloud-services');
+
+const appKey = await secrets.get('my-service/apple-news-app-key');
+const configSecret = await secrets.get('my-service/config', { parseJson: true });
+```
+
+## Creating Secrets
+
+Existing secrets are never overwritten.
+
+```javascript
+await secrets.set('my-service/apple-news-app-key', 'secret-value');
+await secrets.set('my-service/config', { apiKey: 'secret-value' });
+```
+
+CLI usage:
+
+```bash
+npm run secrets:set -- --name=my-service/apple-news-app-key --value='secret-value'
+npm run secrets:set -- --name=my-service/config --json='{"apiKey":"secret-value"}'
+npm run secrets:set -- --name=my-service/apple-news-app-key --file=./secret.txt
+```
+
+If `--name` is omitted, the script uses the current `package.json` name. If no package name is available, it prompts for one.
+
+## Cache Options
+
+By default `secrets.get()` checks a local tmp cache first and writes AWS results back to that cache. There is no TTL. To force AWS:
+
+```javascript
+await secrets.get('my-service/apple-news-app-key', { bypassCache: true });
+```
+
+Set `EIP_SECRETS_CACHE_DIR` or `secrets.cacheDir` to control cache location.
 
 # AWS Lambda Module
 
@@ -228,7 +273,7 @@ The module is designed to handle errors gracefully, providing clear error messag
 
 ## Overview
 
-This module provides functionalities to manage and interact with Content Delivery Networks (CDNs) like Amazon CloudFront and Google Cloud CDN. It includes features for creating invalidations, thereby ensuring that the latest content is served to end-users.
+This module provides a shared client for CDN invalidation requests. CDN invalidation requests are sent through the shared Tools API invalidate endpoint, so callers use one centralized route while the backend implementation remains independently updateable inside the invalidation service.
 
 ## Installation
 
@@ -242,7 +287,7 @@ const cdn = require('eip-cloud-services/cdn');
 
 ### Create a CDN Invalidation
 
-To invalidate cached content in a CDN, use the `createInvalidation` method. This method supports invalidating content in both Amazon CloudFront and Google Cloud CDN.
+To invalidate cached content in a CDN, use the `createInvalidation` method. This sends the request to the shared Tools API invalidate endpoint, which executes the configured provider-specific invalidation in `eip-cdn-invalidator`.
 
 #### Invalidate in Amazon CloudFront
 
@@ -272,11 +317,9 @@ The module is equipped to handle errors gracefully, including validation of inpu
 
 ## Advanced Features
 
-- Supports both Amazon CloudFront and Google Cloud CDN.
 - Validates the key argument to ensure proper formatting.
 - Constructs invalidation paths based on the provided keys.
-- Initializes Google Auth if Google CDN is used.
-- Sends invalidation commands to the CDN client based on the type of CDN and environment.
+- Centralizes invalidation routing through the shared Tools API endpoint.
 
 
 # AWS S3 Module

package/index.js

@@ -4,5 +4,6 @@ exports.s3 = require ( './src/s3' );
 exports.cdn = require ( './src/cdn' );
 exports.lambda = require ( './src/lambda' );
 exports.sqs = require ( './src/sqs' );
+exports.secrets = require ( './src/secrets' );
 exports.mysql = require ( './src/mysql' );
 exports.mysql8 = require ( './src/mysql8' );

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "eip-cloud-services",
-  "version": "1.2.6",
+  "version": "1.4.0",
   "description": "Houses a collection of helpers for connecting with Cloud services.",
   "main": "index.js",
   "scripts": {
-    "test": "mocha"
+    "test": "mocha",
+    "secrets:set": "node scripts/set-secret.js"
   },
   "author": "Oliver Edgington <oliver@edgington.com>",
   "license": "ISC",

package/scripts/set-secret.js

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+
+const fs = require ( 'fs' );
+const readline = require ( 'readline' );
+const secrets = require ( '../src/secrets' );
+
+const parseArgs = ( argv = process.argv.slice ( 2 ) ) => {
+    const args = {};
+    for ( let i = 0; i < argv.length; i++ ) {
+        const arg = argv[ i ];
+        if ( !arg.startsWith ( '--' ) ) continue;
+
+        const [ rawKey, inlineValue ] = arg.slice ( 2 ).split ( '=' );
+        const key = rawKey.replace ( /-([a-z])/g, ( _match, char ) => char.toUpperCase () );
+        if ( inlineValue !== undefined ) {
+            args[ key ] = inlineValue;
+            continue;
+        }
+
+        if ( argv[ i + 1 ] && !argv[ i + 1 ].startsWith ( '--' ) ) {
+            args[ key ] = argv[ i + 1 ];
+            i++;
+            continue;
+        }
+
+        args[ key ] = true;
+    }
+
+    return args;
+};
+
+const prompt = question => new Promise ( resolve => {
+    const rl = readline.createInterface ( {
+        input: process.stdin,
+        output: process.stdout
+    } );
+    rl.question ( question, answer => {
+        rl.close ();
+        resolve ( answer.trim () );
+    } );
+} );
+
+const resolveSecretValue = args => {
+    if ( args.value !== undefined ) return String ( args.value );
+    if ( args.json !== undefined ) return JSON.parse ( String ( args.json ) );
+    if ( args.file ) return fs.readFileSync ( String ( args.file ), 'utf8' );
+    throw new Error ( 'Missing secret value. Use --value, --json, or --file.' );
+};
+
+const main = async () => {
+    const args = parseArgs ();
+    const name = secrets.resolveDefaultSecretName ( { name: args.name } ) || await prompt ( 'Secret name: ' );
+    if ( !name ) throw new Error ( 'Secret name is required.' );
+
+    const value = resolveSecretValue ( args );
+    const response = await secrets.set ( name, value, {
+        region: args.region,
+        description: args.description,
+        cacheDir: args.cacheDir
+    } );
+
+    process.stdout.write ( JSON.stringify ( {
+        name,
+        arn: response.ARN || response.arn || null,
+        versionId: response.VersionId || response.versionId || null
+    }, null, 2 ) );
+    process.stdout.write ( '\n' );
+};
+
+if ( require.main === module ) {
+    main ().catch ( error => {
+        process.stderr.write ( `${ error?.message || error }\n` );
+        process.exitCode = 1;
+    } );
+}
+
+module.exports = {
+    parseArgs,
+    resolveSecretValue
+};

package/src/cdn.js

@@ -1,114 +1,61 @@
-const { CloudFrontClient, CreateInvalidationCommand } = require ( '@aws-sdk/client-cloudfront' );
-const { GoogleAuth } = require ( 'google-auth-library' );
-const { initialiseGoogleAuth } = require ( './gcp' );
 const fs = require ( 'fs' );
 let config = {};
 const configDirPath = `${ process.cwd ()}/config`;
 if ( fs.existsSync ( configDirPath ) && fs.statSync ( configDirPath ).isDirectory () ) {
     config = require ( 'config' ); // require the config directory if it exists
 }
-const packageJson = require ( '../package.json' );
 const { cwd } = require ( 'process' );
 const { log } = config?.s3?.logsFunction ? require ( `${  cwd ()}/${config.s3.logsFunction}` ) : console;
 
-const redis = require('./redis');
+const INVALIDATION_ENDPOINT = process.env.EIP_CDN_INVALIDATE_ENDPOINT || config?.cdn?.invalidateEndpoint || 'https://tools.eip.telegraph.co.uk/v1/invalidate';
+const shouldLog = Boolean ( config?.cdn?.log || config?.cdn?.logs );
 
-/**
- * Create a CDN invalidation for the specified key(s) and environment.
- *
- * @param {string} cdn - The CDN provider to be used (e.g., 'google', 'amazon').
- * @param {string|string[]} key - The key(s) representing the file(s) to invalidate.
- * @param {string} environment - The environment (e.g., 'staging', 'production').
- * @returns {Promise<void>} A promise that resolves when the invalidation is created.
- * @description Creates a CDN invalidation for the specified key(s) in the specified environment.
- * - The `cdn` parameter specifies the CDN provider (e.g., 'google', 'amazon').
- * - The `key` parameter can be a single string or an array of strings representing the file(s) to invalidate.
- * - The `environment` parameter specifies the environment (e.g., 'staging', 'production').
- * - The function validates the `key` argument and throws an error if it is not a string or an array of strings.
- * - The function constructs the invalidation paths based on the provided keys.
- * - The CDN client (either Google or Amazon CloudFront) is used to send the invalidation command.
- * - Returns a promise that resolves when the invalidation is created.
- * - The function initializes Google Auth if Google CDN is used.
- * - If Google CDN is used, the function makes a POST request to Google's 'invalidateCache' endpoint for each provided path.
- * - If Amazon CloudFront is used, the function sends an invalidation command to CloudFront.
- * - If the CDN type is not 'google' or 'amazon', the function throws an error.
- * - The function uses the 'config' module to access the CDN settings based on the CDN provider and environment.
- */
 exports.createInvalidation = async ( cdn, key, environment = 'production' ) => {
-    const cdnSettings = config.cdn[ cdn ][ environment ];
+    const invalidation = normalizeInvalidation ( cdn, key, environment );
 
-    // Ensure paths is an array and sanitize paths
-    const paths = ( Array.isArray ( key ) ? key : [ key ] )
-        .filter ( item => typeof item === 'string' )
-        .map ( item => item.charAt ( 0 ) !== '/' ? '/' + item : item );
-
-    if ( paths.length === 0 ) {
-        throw new Error ( 'Invalid key argument. Expected a string or an array of strings.' );
+    if ( shouldLog ) {
+        log ( `CDN [INVALIDATE][REQUEST]: ${invalidation.cdn} (${invalidation.environment}) ${invalidation.paths.join ( ', ' )}\n` );
     }
 
-    if ( config?.redis?.host ) {
-        const redisKey = `cdn-invalidation:${cdn}:${environment}:${key}`;
-        const redisValue = await redis.get(redisKey);
+    const response = await fetch ( INVALIDATION_ENDPOINT, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json'
+        },
+        body: JSON.stringify ( {
+            cdn: invalidation.cdn,
+            key: invalidation.key,
+            environment: invalidation.environment
+        } )
+    } );
 
-        if(redisValue) {
-            if ( config.cdn.log )
-                log ( `CDN [INVALIDATE]: Invalidation already in progress - skipping: ${paths.map ( path => `https://${cdn}${environment !== 'production' ? '-test' : ''}.eip.telegraph.co.uk${path}` ).join ( ', ' )}\n` );
-            await redis.increment(redisKey);
+    if ( !response.ok ) {
+        const responseText = await response.text ();
 
-            return;
-        }
-        else{
-            await redis.set(redisKey, 1, cdnSettings.type === 'google' ? 300 : 120); // 5 minutes for google, 2 minutes for amazon
-        }
+        throw new Error ( `CDN invalidation request failed (${response.status}): ${responseText}` );
     }
+};
 
-    if ( config.cdn.log )
-        log ( `CDN [INVALIDATE]: ${paths.map ( path => `https://${cdn}${environment !== 'production' ? '-test' : ''}.eip.telegraph.co.uk${path}` ).join ( ', ' )}\n` );
-
-    switch ( cdnSettings.type ) {
-        case 'google':
-            await invalidateGoogleCDN ( cdnSettings, paths );
-            break;
-
-        case 'amazon':
-            await invalidateAmazonCDN ( cdnSettings, paths );
-            break;
+function normalizeInvalidation ( cdn, key, environment ) {
+    const cdnSettings = config?.cdn?.[ cdn ]?.[ environment ];
 
-        default:
-            throw new Error ( `Invalid cdn type: ${cdnSettings.type}` );
+    if ( !cdnSettings ) {
+        throw new Error ( `Missing CDN configuration for ${cdn} (${environment})` );
     }
-};
 
-async function invalidateGoogleCDN ( cdnSettings, paths ) {
-    await initialiseGoogleAuth ();
-
-    const auth = new GoogleAuth ( {
-        scopes: 'https://www.googleapis.com/auth/cloud-platform'
-    } );
+    const paths = ( Array.isArray ( key ) ? key : [ key ] )
+        .filter ( item => typeof item === 'string' )
+        .map ( item => item.charAt ( 0 ) !== '/' ? '/' + item : item );
 
-    const client = await auth.getClient ();
-    const url = `https://compute.googleapis.com/compute/v1/projects/${cdnSettings.projectId}/global/urlMaps/${cdnSettings.urlMapName}/invalidateCache`;
-    const headers = await client.getRequestHeaders ( url );
+    if ( paths.length === 0 ) {
+        throw new Error ( 'Invalid key argument. Expected a string or an array of strings.' );
+    }
 
-    await Promise.all ( paths.map ( path => fetch ( url, {
-        method: 'POST',
-        headers,
-        body: JSON.stringify ( { path } )
-    } ) ) );
+    return {
+        cdn,
+        key,
+        environment,
+        paths,
+        cdnSettings
+    };
 }
-
-async function invalidateAmazonCDN ( cdnSettings, paths ) {
-    const client = new CloudFrontClient ();
-    const command = new CreateInvalidationCommand ( {
-        DistributionId: cdnSettings.distributionId,
-        InvalidationBatch: {
-            CallerReference: `${packageJson.name}-${Date.now ()}`,
-            Paths: {
-                Quantity: paths.length,
-                Items: paths,
-            },
-        },
-    } );
-
-    await client.send ( command );
-}

package/src/s3.js

@@ -50,7 +50,9 @@ const zlib = require ( 'zlib' );
 const crypto = require ( 'crypto' );
 const { cwd } = require ( 'process' );
 const { log } = config?.s3?.logsFunction ? require ( `${  cwd ()}/${config?.s3?.logsFunction}` ) : console;
-const S3 = new S3Client ( { region: 'eu-west-1' } );
+const S3 = new S3Client ( {
+    region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || config?.s3?.region || 'eu-west-1'
+} );
 const { pipeline, Writable } = require ( 'stream' );
 const util = require ( 'util' );
 const pipelineAsync = util.promisify ( pipeline );
@@ -408,4 +410,4 @@ const streamToBuffer = async ( stream ) => {
     await pipelineAsync ( stream, collectorStream );
     
     return Buffer.concat ( chunks );
-};
+};

package/src/secrets.js

@@ -0,0 +1,169 @@
+const { SecretsManagerClient, CreateSecretCommand, DescribeSecretCommand, GetSecretValueCommand } = require ( '@aws-sdk/client-secrets-manager' );
+const crypto = require ( 'crypto' );
+const fs = require ( 'fs' );
+const os = require ( 'os' );
+const path = require ( 'path' );
+
+let config = {};
+const configDirPath = `${ process.cwd ()}/config`;
+if ( fs.existsSync ( configDirPath ) && fs.statSync ( configDirPath ).isDirectory () ) {
+    config = require ( 'config' );
+}
+
+const DEFAULT_REGION = 'eu-west-1';
+const DEFAULT_CACHE_DIR = path.join ( os.tmpdir (), 'eip-cloud-services', 'secrets' );
+
+let clientOverride = null;
+
+const resolveRegion = options => options.region || config?.secrets?.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || DEFAULT_REGION;
+
+const getClient = options => clientOverride || new SecretsManagerClient ( { region: resolveRegion ( options ) } );
+
+const safeCacheFileName = ( secretName, region ) => `${ crypto.createHash ( 'sha256' ).update ( `${ region }:${ secretName }` ).digest ( 'hex' ) }.json`;
+
+const resolveCacheDir = options => options.cacheDir || config?.secrets?.cacheDir || process.env.EIP_SECRETS_CACHE_DIR || DEFAULT_CACHE_DIR;
+
+const resolveCachePath = ( secretName, options = {} ) => path.join ( resolveCacheDir ( options ), safeCacheFileName ( secretName, resolveRegion ( options ) ) );
+
+const readCache = ( secretName, options = {} ) => {
+    if ( options.bypassCache ) return null;
+
+    const cachePath = resolveCachePath ( secretName, options );
+    if ( !fs.existsSync ( cachePath ) ) return null;
+
+    const cached = JSON.parse ( fs.readFileSync ( cachePath, 'utf8' ) );
+    return cached.value;
+};
+
+const writeCache = ( secretName, value, options = {} ) => {
+    if ( options.cache === false ) return;
+
+    const cachePath = resolveCachePath ( secretName, options );
+    fs.mkdirSync ( path.dirname ( cachePath ), { recursive: true, mode: 0o700 } );
+    fs.writeFileSync ( cachePath, JSON.stringify ( {
+        secretName,
+        region: resolveRegion ( options ),
+        cachedAt: Date.now (),
+        value
+    } ), { mode: 0o600 } );
+};
+
+const parseSecretValue = ( value, options = {} ) => {
+    if ( !options.parseJson || typeof value !== 'string' ) return value;
+
+    try {
+        return JSON.parse ( value );
+    }
+    catch ( _error ) {
+        return value;
+    }
+};
+
+const stringifySecretValue = value => typeof value === 'string' ? value : JSON.stringify ( value );
+
+/**
+ * Resolve the default secret name from an explicit value or the current package name.
+ *
+ * @param {object} [options={}] Options.
+ * @param {string} [options.name] Explicit secret name.
+ * @param {string} [options.cwd=process.cwd()] Directory containing package.json.
+ * @returns {string} The resolved secret name, or an empty string when none can be resolved.
+ */
+exports.resolveDefaultSecretName = ( options = {} ) => {
+    if ( options.name ) return String ( options.name ).trim ();
+
+    const packagePath = path.join ( options.cwd || process.cwd (), 'package.json' );
+    if ( !fs.existsSync ( packagePath ) ) return '';
+
+    try {
+        const packageJson = JSON.parse ( fs.readFileSync ( packagePath, 'utf8' ) );
+        return typeof packageJson.name === 'string' ? packageJson.name.trim () : '';
+    }
+    catch ( _error ) {
+        return '';
+    }
+};
+
+/**
+ * Check whether a secret exists in AWS Secrets Manager.
+ *
+ * @param {string} secretName Secret name or ARN.
+ * @param {object} [options={}] Options.
+ * @param {string} [options.region] AWS region.
+ * @returns {Promise<boolean>} True when the secret exists, false when not found.
+ */
+exports.exists = async ( secretName, options = {} ) => {
+    try {
+        await getClient ( options ).send ( new DescribeSecretCommand ( { SecretId: secretName } ) );
+        return true;
+    }
+    catch ( error ) {
+        if ( error?.name === 'ResourceNotFoundException' ) return false;
+        throw error;
+    }
+};
+
+/**
+ * Get a secret value, checking the local tmp cache first unless bypassed.
+ *
+ * @param {string} secretName Secret name or ARN.
+ * @param {object} [options={}] Options.
+ * @param {boolean} [options.bypassCache=false] Force AWS lookup instead of tmp cache.
+ * @param {boolean} [options.cache=true] Write AWS results to tmp cache.
+ * @param {boolean} [options.parseJson=false] Parse JSON string secrets when possible.
+ * @param {string} [options.region] AWS region.
+ * @param {string} [options.cacheDir] Custom cache directory.
+ * @returns {Promise<string|object>} Secret value.
+ */
+exports.get = async ( secretName, options = {} ) => {
+    const cached = readCache ( secretName, options );
+    if ( cached !== null && cached !== undefined ) return parseSecretValue ( cached, options );
+
+    const response = await getClient ( options ).send ( new GetSecretValueCommand ( { SecretId: secretName } ) );
+    const value = response.SecretString || ( response.SecretBinary ? Buffer.from ( response.SecretBinary ).toString ( 'utf8' ) : '' );
+    writeCache ( secretName, value, options );
+
+    return parseSecretValue ( value, options );
+};
+
+/**
+ * Create a new secret in AWS Secrets Manager. Existing secrets are never overwritten.
+ *
+ * @param {string} secretName Secret name.
+ * @param {string|object} value Secret value.
+ * @param {object} [options={}] Options.
+ * @param {string} [options.description] Secret description.
+ * @param {string} [options.region] AWS region.
+ * @param {string} [options.cacheDir] Custom cache directory.
+ * @returns {Promise<object>} AWS CreateSecret response.
+ */
+exports.set = async ( secretName, value, options = {} ) => {
+    if ( await exports.exists ( secretName, options ) ) {
+        throw new Error ( `Secret already exists: ${ secretName }` );
+    }
+
+    const secretString = stringifySecretValue ( value );
+    const response = await getClient ( options ).send ( new CreateSecretCommand ( {
+        Name: secretName,
+        SecretString: secretString,
+        Description: options.description
+    } ) );
+    writeCache ( secretName, secretString, options );
+
+    return response;
+};
+
+exports.__private = {
+    parseSecretValue,
+    readCache,
+    resolveCachePath,
+    safeCacheFileName,
+    stringifySecretValue,
+    writeCache,
+    setClient: client => {
+        clientOverride = client;
+    },
+    clearClient: () => {
+        clientOverride = null;
+    }
+};