ac-awssecrets 1.1.5 → 2.0.0

package/index.js

@@ -1,182 +1,172 @@
-const _ = require('lodash')
-const async = require('async')
-const AWS = require('aws-sdk')
+const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager");
+const { fromIni } = require("@aws-sdk/credential-providers")
+
+const testConfig = require('./test/config')
 
 /**
- * @param aws OBJ object with aws configuration data
- */
+* Replaces configuration variables with secrets
 
-const awsSecrets = () => {
-  const loadSecrets = (params, cb) => {
-    const multiSecrets = _.get(params, 'multisecrets', [])
-    const secrets = _.get(params, 'secrets', [])
-    let config = _.get(params, 'config', {}) // the config object -> will be changed by reference
-    const environment = _.get(config, 'environment', 'development')
-
-    const region = _.get(params, 'aws.region', 'eu-central-1')
-    const endpoint = _.find(awsEndpoints, { region })
-    const client = new AWS.SecretsManager({
-      accessKeyId: _.get(params, 'aws.accessKeyId'),
-      secretAccessKey: _.get(params, 'aws.secretAccessKey'),
-      region: region,
-      endpoint: _.get(endpoint, 'endpoint')
-    })
-
-    let result = []
-    async.series({
-      fetchPlacholders: (done) => {
-        if (!_.size(multiSecrets)) return done()
-        // some keys can have multiple entries (e.g. cloudfrontCOnfigs can have 1 - n entries)
-        // we have to fetch them first from a secret and add them to the secrets to fetch
-
-        async.each(multiSecrets, (secret, itDone) => {
-          let secretName = (config.environment === 'test' ? 'test.' : '') + _.get(secret, 'name')
-
-          client.getSecretValue({ SecretId: secretName }, function(err, data) {
-            if (err) {
-              if (_.get(secret, 'ignoreInTestMode')) return itDone()
-              if (_.get(secret, 'ignoreIfMissing')) return itDone() // this is an optional key
-
-              console.error('Fetching secret %s failed', secretName, err)
-              return itDone({ message: err, additionalInfo: { key: secretName } })
-            }
-            if (!_.get(data, 'SecretString')) {
-              console.warn('Secret %s NOT avaialble', secretName, _.get(data, 'SecretString'))
-              return itDone()
-            }
+KEY is the variable name
+NAME is the name of the secret 
 
-            let value
-            try {
-              value = JSON.parse(_.get(data, 'SecretString'))
-            }
-            catch (e) {
-              value = _.get(data, 'SecretString.values')
-            }
+OPT
+serverName
 
-            try {
-              value = JSON.parse(_.get(value, 'values'))
-            }
-            catch (e) {
-              return done({ message: 'placeHolderSecrets_valuesInvalid', additionalInfo: { key: secret.key } })
-            }
+MULTISECRETS
+Multisecrets -> the secret contains a list of secrets that should be fetched
+*
+* TESTMODES
+* 3 -> use secrets from testConfig
+*/
 
-            // value should be an array of keys
-            _.forEach(value, (item) => {
-              secrets.push({
-                key: secret.key,
-                name: item,
-                type: 'arrayObject'
-              })
-            })
-            return itDone()
-          })
-        }, done)
-      },
-      fetchSecrets: (done) => {
-        if (!_.size(secrets)) return done()
-        async.each(secrets, (secret, itDone) => {
-          if (environment === 'test' && _.get(secret, 'ignoreInTestMode')) return itDone()
-          // key is the local configuration path
-          let key = _.get(secret, 'key')
-          // secret name is the name used to fetch the secret
-          let secretName = (config.environment === 'test' ? 'test.' : '') + _.get(secret, 'name') + (_.get(secret, 'suffix') ? '.' + _.get(secret, 'suffix') : '')
-
-          client.getSecretValue({ SecretId: secretName }, function(err, data) {
-            if (err) {
-              if (_.get(secret, 'ignoreIfMissing')) return itDone() // this is an optional key
-
-              console.error('Fetching secret %s failed', secretName, _.get(err, 'message', err))
-              return itDone({ message: _.get(err, 'message', err), additionalInfo: { key: secretName } })
-            }
-            if (!_.get(data, 'SecretString')) {
-              console.warn('Secret %s NOT avaialble', secretName, _.get(data, 'SecretString'))
-              return itDone()
-            }
+const awsSecrets = () => {
 
-            let value
-            try {
-              value = JSON.parse(_.get(data, 'SecretString'))
-
-              // if value is prefixed with JSON -> parse the value
-              if (_.get(value, 'valueHasJSON')) {
-                _.forEach(value, (val, key) => {
-                  if (_.startsWith(val, 'JSON:')) {
-                    try {
-                      _.set(value, key, JSON.parse(val.substr(5)))
-                    }
-                    catch (e) {
-                      throw e
-                    }
-                  }
-                })
-                // remove that entry
-                _.unset(value, 'valueHasJSON')
-              }
-            }
-            catch (e) {
-              value = _.get(data, 'SecretString')
-            }
+  const getKey = (obj, key) => key.split('.').reduce((acc, cur) => acc[cur], obj)
 
-            // make sure boolean values are converted (from string)
-            value = _.mapValues(value, (val) => {
-              if (val === 'true') return true
-              else if (val === 'false') return false
-              else return val
-            })
-            let existingValue = _.get(config, key, {})
+  const setKey = (obj, key, value ) => {
+    const [head, ...rest] = key.split('.')
+    !rest.length
+        ? obj[head] = value
+        : setKey(obj[head], rest.join('.'), value)
+  }
 
-            if (secret.servers) {
-              if (_.isBoolean(secret.servers)) {
-                // LEGACY SUPPORT FOR OLD NOTATION - DEPRECATED - DO NOT USE ANY LONGER
-                existingValue = _.find(_.get(config, key + '.servers', []), { server: secret.serverName })
-              }
-              else {
-                // NEW NOTATION AS OBJECT
-                let match = {}
-                _.set(match, _.get(secret.servers, 'identifier'), _.get(secret.servers, 'value'))
-                existingValue = _.find(_.get(config, key, []), match)      
-              }
-            }
-            if (_.get(secret, 'type') === 'array') {
-              let array = []
-              _.forEach(value, (val) => {
-                array.push(val)
-              })
-              value = _.concat(existingValue, array)
-            }
 
-            if (_.get(secret, 'type') === 'arrayObject') {
-              existingValue.push(value)
-            }
-            else {
-              let setFresh
-              if (_.isEmpty(existingValue)) setFresh = true
-              _.merge(existingValue, value)
-              // setFresh -> this property/path has never existed
-              if (setFresh) _.set(config, key, existingValue)
-            }
+  const functionName = 'ac-awsSecrets'.padEnd(15)
+  const loadSecrets = async({ secrets = [], multisecrets = [], config = {}, profile = process.env['profile'], testMode = 0, debug = false } = {}) => {
+    const environment = config?.environment || 'development'
+
+    const awsConfig = {
+      region: 'eu-central-1'
+    }
+    // credentials are determined from Lambda role.
+    // But you can also use a set profile 
+    if (profile) {
+      console.error('%s | Using AWS profile | %s', functionName, profile)
+      awsConfig.credentials = fromIni({ profile })
+    }
+    const client = new SecretsManagerClient(awsConfig)
+
+  
+    const getSecret = async({ secret }) => {
+      const secretName = (environment === 'test' ? 'test.' : '') + secret?.name + (secret?.suffix ? '.' + secret?.suffix : '')
+
+      // TESTMODE
+      if (testMode === 3) {
+        // fetch from availableSecrets
+        let found = testConfig.availableSecrets.find(item => item.name === secret.name)
+        secret.value = found?.value
+      }
+      else {
+        const command = new GetSecretValueCommand({
+          SecretId: secretName
+        })
+        try {
+          const response = await client.send(command)
+          if (response?.SecretString) {
+            secret.value = JSON.parse(response?.SecretString)
+          }
+        }
+        catch(e) {
+          console.error('%s | %s | %s', functionName, secretName, e?.message)
+        }
+      }
+      return secret
+    }
 
-            if (_.get(secret, 'log')) {
-              console.log(_.repeat('.', 90))
-              console.warn(key, existingValue)
-              console.warn(_.get(config, key))
-              console.warn(_.repeat('.', 90))
+    const fetchSecrets = async({ secrets }) => {
+      // filter out secrets with ignoreInTestMode = true
+      if (environment === 'test') {
+        secrets = secrets.filter(secret => !secret.ignoreInTestMode)
+      }
+      return Promise.all(secrets.map(secret => getSecret({ secret })))
+    }
+
+    // fetch placeholder
+    if (multisecrets.length > 0) {
+      // some keys can have multiple entries (e.g. cloudfrontCOnfigs can have 1 - n entries)
+      // we have to fetch them first from a secret and add them to the secrets to fetch 
+      let secretsToAdd = await fetchSecrets({ secrets: multisecrets })
+      // iterate each multisecret and add the values as new secrets
+      secretsToAdd.forEach(secadd => {
+        let items = JSON.parse(secadd?.value?.values) || []
+        if (typeof items !== 'object' || items.length < 1) {
+          console.error('%s | %s | MultiSecret has no valid property values', functionName, secadd.name)
+          throw new Error('MultiSecret has no valid property values')
+        }
+        items.forEach(item => {
+          let p = {
+            key: secadd.key,
+            name: item,
+            type: 'arrayObject' // multisecrets contain multiple secrets that belong to the same config property (which is an array of objects)
+          }
+          secrets.push(p)
+        })
+      })
+    }
+
+    if (secrets.length > 0) {
+      await fetchSecrets({ secrets })
+      for (const secret of secrets) {
+        let existingValue = getKey(config, secret.key) || {}
+        let value = secret?.value
+
+        // convert values
+        if (typeof value === 'object') {
+          Object.keys(value).forEach((key) => {
+            let val = value[key]
+            if (val === 'true') val = true
+            else if (val === 'false') val = false
+            else if (typeof val === 'string' && val.startsWith('JSON:')) {
+              try {
+                val = JSON.parse(val.substring(5))
+              }
+              catch(e) {
+                console.error('%s | %s | JSON could not be parsed %j', functionName, secret.name, val)
+                throw new Error('invalidJSON')
+              }
             }
-
-            result.push({ key, name: _.get(secret, 'name', '-') })
-            return itDone()
+            value[key] = val
           })
-        }, done)
+        }
+
+        if (secret.servers) {
+          if (typeof secret.servers === 'boolean') {
+            let servers = existingValue?.servers || []
+            config[secret.key].servers = servers.map(server => {
+              if (server.server === secret.serverName) {
+                server = { ...server, ...value }
+              }
+              return server
+            })
+          }
+          else {
+            // NEW NOTATION AS OBJECT
+            /* TODO: Probably not used anywhere, so legacy is ok
+            let match = {}
+            _.set(match, _.get(secret.servers, 'identifier'), _.get(secret.servers, 'value'))
+            existingValue = _.find(_.get(config, key, []), match)   
+            */
+          }
+        }
+        else if (secret?.type === 'arrayObject') {
+          existingValue.push(value)
+          setKey(config, secret.key, existingValue)
+        }
+        else {
+          if (Object.keys(existingValue).length === 0) {
+            setKey(config, secret.key, {})
+          }
+          existingValue = { ...existingValue, ...value }
+          setKey(config, secret.key, existingValue)
+        }
+
+        if (secret?.log || debug) {
+          console.log('%s | %s | %j', functionName, secret?.name, existingValue)
+        }
       }
-    }, (err) => {
-      return cb(err, _.orderBy(result, 'key'))
-    })
+    }  
   }
 
-  const awsEndpoints = [
-    { region: 'eu-central-1', endpoint: 'https://secretsmanager.eu-central-1.amazonaws.com' }
-  ]
-
   return {
     loadSecrets
   }

package/package.json

@@ -3,19 +3,21 @@
   "author": "Mark Poepping (https://www.admiralcloud.com)",
   "license": "MIT",
   "repository": "admiralcloud/ac-awssecrets",
-  "version": "1.1.5",
+  "version": "2.0.0",
   "dependencies": {
-    "async": "^3.2.3",
-    "aws-sdk": "^2.1101.0",
-    "lodash": "^4.17.21"
+    "@aws-sdk/client-secrets-manager": "^3.279.0",
+    "@aws-sdk/credential-providers": "^3.279.0"
   },
   "devDependencies": {
-    "ac-semantic-release": "^0.2.7",
+    "ac-semantic-release": "^0.3.5",
+    "chai": "^4.3.7",
     "eslint": "8.x",
-    "mocha": "^9.2.2"
+    "mocha": "^10.2.0",
+    "nyc": "^15.1.0"
   },
   "scripts": {
-    "test": "mocha --reporter spec"
+    "test": "mocha --reporter spec",
+    "coverage": "./node_modules/nyc/bin/nyc.js report --reporter=lcov --reporter=text"
   },
   "engines": {
     "node": ">=8.0.0"

package/test/config.js

@@ -0,0 +1,130 @@
+const config = {
+  configVar1: {
+    c1: false
+  },
+  configVar2: {
+    servers: [
+      { server: 'cacheRead', host: 'localhost', port: 6379 },
+    ]
+  },
+  configVar4: {
+    api: {
+      port: 90
+    }
+  },
+  configVar5: {
+    path: {
+      cookie: false
+    }
+  },
+  aws: {
+    accessKeys: []
+  }
+}
+
+const secrets = [
+  { key: 'configVar1', name: 'simple' },
+  { key: 'configVar2', name: 'server', servers: true, serverName: 'cacheRead' },
+  { key: 'configVar4', name: 'json' },
+  { key: 'configVar5.path', name: 'path' },
+  { key: 'configVar6', name: 'notExistingLocally' },
+]
+
+const availableSecrets = [{
+  key: 'configVar1',
+  name: 'simple',
+  value: {
+    c1: 'true',
+    c2: 123,
+    c3: 'abc'
+  },
+  log: true
+}, {
+  key: 'configVar2',
+  name: 'server',
+  value: {
+    port: 6360,
+    host: 'myRedisHost'
+  }
+},
+{
+  key: 'configVar3',
+  name: 'noSecret'
+}, 
+{
+  key: 'configVar4',
+  name: 'json',
+  value: {
+    api: 'JSON:{"url":"https://api.admiralcloud.com"}',
+    valueHasJSON: true
+  }
+},
+{
+  key: 'errorVar1',
+  name: 'invalidJSON',
+  value: {
+    api: 'JSON:abc',
+    valueHasJSON: true
+  }
+},
+{
+  key: 'configVar5.path',
+  name: 'path',
+  value: {
+    cookie: true
+  }
+},
+{
+  key: 'configVar6',
+  name: 'notExistingLocally',
+  value: {
+    prop1: 123,
+    prop2: 'abc'
+  }
+},
+{ 
+  key: 'aws.accessKeys', 
+  name: 'aws.accessKeyConfigs',
+  value: {
+    values: '["aws.key1", "aws.key2"]'
+  }
+},
+{ 
+  key: 'aws.failedKeys', 
+  name: 'aws.failedKeysConfig',
+  value: {
+    values: 123
+  }
+},
+{
+  key: 'aws.key1',
+  name: 'aws.key1',
+  value: {
+    accessKeyId: 'awsKey1',
+    secretAccessKey: 'awsSecret1'
+  }
+},{
+  key: 'aws.key2',
+  name: 'aws.key2',
+  value: {
+    accessKeyId: 'awsKey2',
+    secretAccessKey: 'awsSecret2'
+  }
+}]
+
+const multisecrets = [
+  { key: 'aws.accessKeys', name: 'aws.accessKeyConfigs' }
+]
+
+const multisecretsFail = [
+  { key: 'aws.failedKeys', name: 'aws.failedKeysConfig' } 
+]
+
+module.exports = {
+  config,
+  availableSecrets,
+  multisecretsFail,
+  secrets,
+  multisecrets
+}
+

package/test/test.js

@@ -1,7 +1,101 @@
-//const awsSecrets = require('../index')
+const { expect } = require('chai')
 
-describe('Tests', () => {
-  it('Should work', done => {
-    return done()
+
+ const awsSecrets = require('../index')
+
+const testConfig = require('./config')
+
+const multisecrets = testConfig?.multisecrets
+const config = testConfig.config
+let secrets = testConfig.secrets
+
+// HELPER for console.log checks
+const captureStream = (stream) => {
+  let oldWrite = stream.write
+  var buf = ''
+  stream.write = (chunk) => {
+    buf += chunk.toString()
+    oldWrite.apply(stream, arguments)
+  }
+
+  return {
+    unhook: () => {
+     stream.write = oldWrite
+    },
+    captured: () => {
+      return buf
+    }
+  }
+}
+
+
+describe('Reading secrets', () => {
+  it('Read secrets', async() => {
+    await awsSecrets.loadSecrets({ secrets, config, multisecrets, testMode: 3 })
+    //console.log(18, config)
+  })
+
+  it('Check configVar1', async() => {
+    const expected = secrets.find(item => item.key === 'configVar1')
+    expect(config.configVar1).to.have.property('c1', true)
+    expect(config.configVar1).to.have.property('c2', expected.value.c2)
+    expect(config.configVar1).to.have.property('c3', expected.value.c3)
+
+  })
+
+  it('Check server', async() => {
+    const expected = secrets.find(item => item.key === 'configVar2')
+    expect(config.configVar2.servers[0]).to.have.property('port', expected.value.port)
+  })
+
+  it('Check JSON', async() => {
+    expect(config.configVar4.api).to.have.property('url', 'https://api.admiralcloud.com')
+  })
+
+  it('Check path', async() => {
+    expect(config.configVar5.path).to.have.property('cookie', true)
+  })
+
+  it('Check non existing local config', async() => {
+    expect(config.configVar6).to.have.property('prop1', 123)
+    expect(config.configVar6).to.have.property('prop2', 'abc')
+  })
+
+  it('Check multisecrets', async() => {
+    //console.log(50, config.aws.accessKeys)
+    expect(config.aws.accessKeys).to.have.length(2)
+    expect(config.aws.accessKeys[0]).to.have.property('accessKeyId', 'awsKey1')
+    expect(config.aws.accessKeys[1]).to.have.property('accessKeyId', 'awsKey2')
+  })
+})
+
+describe('Check errors', () => {
+  it('Check invalid JSON', async() => {
+    let errorSecrets = [
+      { name: 'invalidJSON', key: 'errorVar1' }
+    ]
+    try { 
+      await awsSecrets.loadSecrets({ secrets: errorSecrets, config, testMode: 3 })
+    }
+    catch(e) {
+      expect(e).to.be.an('error')
+      expect(e).to.have.property('message', 'invalidJSON')
+    }
   })
 })
+
+
+describe('Misc', () => {
+  var hook
+  beforeEach(function(){
+    hook = captureStream(process.stdout)
+  })
+  afterEach(function(){
+    hook.unhook()
+  })
+
+  it('Read secrets with debug mode', async() => {
+    await awsSecrets.loadSecrets({ secrets, config, multisecrets, testMode: 3, debug: true })
+    expect(hook.captured()).to.include('ac-awsSecrets')
+  })
+})