configorama 0.6.1 → 0.6.3

package/cli.js

@@ -4,6 +4,7 @@ const fs = require('fs')
 const minimist = require('minimist')
 const Configorama = require('./src/main')
 const deepLog = require('./src/utils/deep-log')
+const { logHeader } = require('./src/utils/logs')
 
 // Parse command line arguments
 const argv = minimist(process.argv.slice(2), {
@@ -79,7 +80,7 @@ const options = {
 }
 
 if (options.dynamicArgs.verbose) {
-  console.log('───────────── Input Options ──────────────────────')
+  logHeader('Config Input Options')
   const dynamicArgs = options.dynamicArgs || {}
   const { 
     _, 

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "configorama",
-  "version": "0.6.1",
+  "version": "0.6.3",
   "description": "Variable support for configuration files",
-  "main": "lib/index.js",
+  "main": "src/index.js",
   "files": [
     "cli.js",
     "src",
@@ -34,8 +34,10 @@
     "url": "https://github.com/DavidWells/configorama"
   },
   "dependencies": {
+    "@davidwells/box-logger": "^1.0.10",
     "@iarna/toml": "^2.2.5",
     "dot-prop": "^5.3.0",
+    "env-stage-loader": "^1.1.1",
     "find-up": "^3.0.0",
     "git-url-parse": "^14.0.0",
     "js-yaml": "^3.14.1",
@@ -60,6 +62,7 @@
     "lodash.split": "^4.4.2",
     "minimist": "^1.2.8",
     "promise.prototype.finally": "^3.1.8",
+    "safe-chalk": "^1.0.0",
     "sync-rpc": "^1.3.6",
     "traverse": "^0.6.8"
   },

package/src/main.js

@@ -12,6 +12,8 @@ const promiseFinallyShim = require('promise.prototype.finally').shim()
 const findUp = require('find-up')
 const traverse = require('traverse')
 const dotProp = require('dot-prop')
+const chalk = require('./utils/chalk')
+
 /* Default Value resolvers */
 const getValueFromString = require('./resolvers/valueFromString')
 const getValueFromNumber = require('./resolvers/valueFromNumber')
@@ -48,6 +50,8 @@ const { encodeUnknown, decodeUnknown } = require('./utils/unknownValues')
 const { mergeByKeys } = require('./utils/mergeByKeys')
 const { arrayToJsonPath } = require('./utils/arrayToJsonPath')
 const { findNestedVariables } = require('./utils/find-nested-variables')
+const { makeBox } = require('@davidwells/box-logger')
+const { logHeader } = require('./utils/logs')
 /**
  * Maintainer's notes:
  *
@@ -160,6 +164,26 @@ class Configorama {
       this.originalString = fileContents
       // Keep a copy
       this.originalConfig = cloneDeep(configObject)
+
+      const useDotEnv = this.originalConfig.useDotenv || this.originalConfig.useDotEnv
+      if ((useDotEnv && useDotEnv === true) || this.opts.useDotEnvFiles) {
+        const loadStageEnv = require('env-stage-loader')
+        let providerStage
+        if (this.originalConfig && this.originalConfig.provider && this.originalConfig.provider.stage) {
+          providerStage = this.originalConfig.provider.stage
+          // @TODO check value to see if variable and needs pre-resolving to resolve stage vars
+        }
+        const stage = this.opts.stage || providerStage || 'dev'
+        /* Load env variables into process.env */
+        const values = loadStageEnv({
+          // silent: true,
+          // debug: true,
+          env: stage,
+          // defaultEnv: 'prod',
+          // ignoreFiles: ['.env']
+        })
+      }
+
       // Set configPath for file references
       this.configPath = fileDirectory
     }
@@ -407,7 +431,7 @@ class Configorama {
     }
 
     if (VERBOSE) {
-      console.log('───────────── Input Config ──────────────────────')
+      logHeader('Config Input before processing')
       console.log()
       deepLog(this.originalConfig)
       console.log()
@@ -415,60 +439,229 @@ class Configorama {
 
     if (VERBOSE || showFoundVariables) {
       const foundVariables = []
-      let loggedHeader = false
+      const variableData = {}
+      let matchCount = 1
       traverse(this.originalConfig).forEach(function (rawValue) {
         if (typeof rawValue === 'string' && rawValue.match(variableSyntax)) {
           const configValuePath = this.path.join('.')
           if (configValuePath.endsWith('Fn::Sub')) {
             return
           }
-
-
-          if (!loggedHeader) {
-            console.log('───────────── Variables Detected ──────────────────────')
-            console.log()
-            loggedHeader = true
-          }
       
           const nested = findNestedVariables(rawValue, variableSyntax, variablesKnownTypes, configValuePath)
           /*
-          console.log(nested)
+          console.log('traverse nested result', nested)
           /** */
           
-          console.log(`▷ Path: ${configValuePath}`)
-          console.log('\n  Key/value:')
-          console.log(`  ${configValuePath}: ${rawValue}`)
-          if (nested.length > 0) {
-            const nestedCount = nested.length - 1
-            console.log('\n  Variable:')
-            console.log(`  ${nested[nested.length - 1].fullMatch}`)
-            
-            if (nestedCount) {
-              console.log(`\n  Contains ${nestedCount} nested values.`)
+          // console.log(`▷ Path: ${configValuePath}`)
+          // console.log('\n  Key/value:')
+          // console.log(`  ${configValuePath}: ${rawValue}`)
+          const lastItem = nested[nested.length - 1]
+          const lastKeyPath = this.path[this.path.length - 1]
+          const itemKey = (lastKeyPath.match(/[\d+]$/)) ? `${this.path[this.path.length - 2]}[${lastKeyPath}]` : lastKeyPath
+          const key = lastItem.fullMatch
+          const varData = {
+            path: configValuePath,
+            key: itemKey,
+            value: rawValue,
+            variable: lastItem.fullMatch,
+            isRequired: false,
+            defaultValue: undefined,
+            matchIndex: matchCount++,
+            // hasFallback: false,
+            resolveOrder: [],
+            resolveDetails: nested,
+          }
+
+          function calculateResolveOrder(item) {
+            if (item && item.fallbackValues) {
+              let hasResolvedFallback
+              const order = ([item.valueBeforeFallback]).concat(item.fallbackValues.map((f, i) => {
+                if (f.fallbackValues) {
+                  const [nestedOrder, nestedResolvedFallback] = calculateResolveOrder(f)
+                  if (!hasResolvedFallback && nestedResolvedFallback) {
+                    hasResolvedFallback = nestedResolvedFallback
+                  }
+                  return nestedOrder // Return just the order part
+                }
+                if (!hasResolvedFallback && f.isResolvedFallback) {
+                  hasResolvedFallback = f.stringValue
+                }
+                return `${f.stringValue || f.variable}${f.isResolvedFallback ? ' (Resolved default fallback)' : ''}`
+              })).flat()
+              
+              return [order, hasResolvedFallback]
             }
-  
-            // non mutate remove last
-            const removeLast = (nested.length > 1) ? nested.slice(0, -1) : nested
-            console.log()
-            removeLast.forEach((v) => {
-              if (v.hasFallback) {
-                console.log('  Resolve order:')
-                console.log(`    1. ${v.valueBeforeFallback}`)
-                v.fallbackValues.forEach((f, i) => {
-                  console.log(`    ${i + 2}. ${f.fullMatch}${f.isFallback ? ' (Fallback string value)' : ''}`)
-                })
-              }
-            })
+            return [[item.variable], false] // Return array instead of just the value
           }
-          console.log()
+
+          const [resolveOrder, hasResolvedFallback] = calculateResolveOrder(lastItem)
+          varData.resolveOrder = resolveOrder
+          
+          if (hasResolvedFallback) {
+            varData.defaultValue = hasResolvedFallback
+          }
+
+
+          if (!varData.defaultValue) {
+            varData.isRequired = true
+          }
+      
+
+          if (varData.resolveOrder.length > 1) {
+            varData.hasFallback = true
+          }
+
+          variableData[key] = (variableData[key] || []).concat(varData)
+
           foundVariables.push(rawValue)
         }
       })
 
-      console.log(`───────────── Found ${foundVariables.length} Variables ──────────────────────`)
-      console.log()
-      deepLog(foundVariables)
-      console.log()
+
+      if (!foundVariables.length) {
+        logHeader('No Variables Found in Config')
+        if (this.configFilePath) {
+          console.log(`File: ${this.configFilePath}`)
+        }
+        
+        console.log(`\nVariable syntax: `, variableSyntax)
+
+        const varTypes = Object.keys(this.variableTypes)
+        if (varTypes.length) {
+          const exclude = ['fallthrough', 'deep']
+          console.log('\nAllowed variable types:')
+          varTypes.filter((v) => v.type !== 'fallthrough').forEach((v) => {
+            const vData = this.variableTypes[v]
+            if (exclude.includes(vData.type)) {
+              return
+            }
+            console.log(`  - ${vData.type}: `, vData.match)
+          })
+        }
+        console.log()
+      }
+
+      // make foundVariables array unique
+      const finalFoundVariables = [...new Set(foundVariables)]
+      if (finalFoundVariables.length > 0) {
+        const varKeys = Object.keys(variableData)
+        const fileName = this.configFilePath ? ` in ${this.configFilePath}` : ''
+
+        logHeader(`Found ${varKeys.length} Variables${fileName}`)
+
+        // deepLog('variableData', variableData)
+
+        if (varKeys.length) {
+          console.log()
+          const longestKey = varKeys.reduce((acc, k) => {
+            return Math.max(acc, k.length)
+          }, 0)
+          console.log(varKeys.map((k) => {
+            const placesWord = variableData[k].length > 1 ? 'places' : 'place'
+            return `- ${k.padEnd(longestKey).padEnd(longestKey + 10)} referenced ${variableData[k].length} ${placesWord}`
+          }).join('\n'))
+          console.log()
+        }
+
+        logHeader('Variable Details')
+    
+        const indent = ''
+        varKeys.forEach((key, i) => {
+          const variableInstances = variableData[key]
+      
+          const firstInstance = variableInstances[0]
+
+          let requiredText = ''
+          let defaultValueSrc = ''
+          if (!firstInstance.defaultValue) {
+            // console.log('no default value', firstInstance)
+            /* Check if the fallback variable is a self reference */
+            const hasDotPropOrSelf = variableInstances.reduce((acc, v) => {
+              const dotProp = v.resolveDetails.find((d) => d.varType === 'dot.prop')
+              if (dotProp) {
+                acc.push(dotProp)
+              }
+              if (v.resolveDetails && v.resolveDetails.length === 1 && v.resolveDetails[0].varType === 'self:') {
+                // console.log('dot.prop', v.resolveDetails)
+                acc.push(v.resolveDetails[0])
+              }
+              return acc
+            }, [])
+            // console.log('hasDotPropOrSelf', hasDotPropOrSelf)
+            if (!hasDotPropOrSelf.length) {
+              requiredText = '[Required] '
+            } else {
+              const fallBackValues = variableInstances.filter((v) => v.resolveDetails.find((d) => d.hasFallback)).map((v) => v.resolveDetails)
+              // console.log('fallBackValues', fallBackValues)
+              if (fallBackValues.length) {
+                // console.log('fallBackValues.resolveDetails', fallBackValues)
+              }
+
+              const cleanPath = hasDotPropOrSelf[0].variable.replace('self:', '')
+              defaultValueSrc = cleanPath
+              // Find the dot prop value in the original config
+              const dotPropValue = dotProp.get(this.originalConfig, cleanPath)
+              // console.log('dotPropValue', dotPropValue)
+              if (typeof dotPropValue !== 'undefined') {
+                requiredText = ''
+                const niceString = typeof dotPropValue === 'object' ? JSON.stringify(dotPropValue) : dotPropValue
+                // truncate niceString to 100 characters
+                const truncatedString = niceString.length > 100 ? niceString.substring(0, 90) + '...' : niceString
+                firstInstance.defaultValue = truncatedString
+              }
+            }
+            //this.originalConfig[key] = undefined
+          }
+          const spacing = '           '
+          const titleText = `Variable:${spacing}`
+          const reqText = (requiredText) ? `${chalk.red.bold(requiredText)}\n` : ''
+          let varMsg = `${reqText}`
+          const VALUE_HEX = '#899499' // '#708090'
+          const keyChalk = chalk.whiteBright
+          const valueChalk = chalk.hex(VALUE_HEX)
+
+          if (firstInstance.defaultValue) {
+            const defaultValueText = `${indent}${keyChalk(`DefaultValue:`.padEnd(titleText.length, ' '))}`
+            // ensure padding is even 
+            varMsg += `${defaultValueText} ${valueChalk(firstInstance.defaultValue)}`
+          }
+
+          if(defaultValueSrc) {
+            varMsg += `\n${indent}${keyChalk('DefaultValue path:'.padEnd(titleText.length, ' '))} `
+            varMsg += `${valueChalk(defaultValueSrc)}`
+          }
+
+          if (firstInstance.resolveOrder.length > 1) {
+            varMsg += `\n${indent}${keyChalk('Resolve Order:'.padEnd(titleText.length, ' '))}`
+            const resolveOrder = firstInstance.resolveOrder.join(', ')
+            varMsg += ` ${valueChalk(resolveOrder)}`
+          }
+
+          let locationRender = valueChalk(variableInstances[0].path)
+  
+          let locationLabel = `${indent}${keyChalk('Used at:'.padEnd(titleText.length, ' '))}`
+          if (variableInstances.length > 1) {
+            locationRender = `\n${variableInstances.map((v) => valueChalk(`${indent}- ${v.path}`)).join('\n')}`
+            const locationLabelText = `${indent}${keyChalk('Used at:')}`
+            locationLabel = locationLabelText
+          }
+
+          varMsg += `\n${locationLabel} ${locationRender}`
+    
+          // console.log(` ${chalk.bold(key)}`)
+          console.log(makeBox(varMsg, {
+            title: `${key}`,
+            borderColor: 'gray',
+            // style: 'bold',
+            minWidth: 120,
+          }))
+          if(i < varKeys.length - 1) {
+            //console.log()
+          }
+        })
+      }
+    
       /* Exit early if list or info flag is set */
       if (showFoundVariables) {
         process.exit(0)
@@ -571,7 +764,7 @@ class Configorama {
             this.config = mergeByKeys(this.config, '', this.mergeKeys)
           }
           if (VERBOSE) {
-            console.log('───────────── Resolved Config ───────────────────')
+            logHeader('Resolved Configuration value')
             console.log()
             deepLog(this.config)
             console.log()

package/src/parsers/yaml.js

@@ -153,7 +153,9 @@ function preProcess(ymlStr = '') {
 
   /* If have yaml object and vars not wrapped in quotes, wrap them */
   if (ymlStr.match(KEY_OBJECT)) {
-    const hasObjects = matchOutermostBraces(ymlStr)
+    const values = matchOutermostBraces(ymlStr)
+    // console.log('values', values)
+    const hasObjects = values.filter((x) => !x.match(/{{resolve:/))
     // console.log('hasObjects', hasObjects)
     if (hasObjects && hasObjects.length) {
       hasObjects.forEach((txt) => {
@@ -187,6 +189,15 @@ function preProcess(ymlStr = '') {
         }
       })
     }
+    // Automagically wrap CF https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/dynamic-references-ssm.html 
+    const cfParams = values.filter((x) => !x.match(/\s/) && x.match(/{{resolve:/))
+    if (cfParams && cfParams.length) {
+      cfParams.forEach((txt) => {
+        const pat = new RegExp(`([^'"])${txt}([^'"])`, 'g')
+        const fixedText = `$1"${txt}"$2`
+        ymlStr = ymlStr.replace(pat, fixedText)
+      })
+    }
   }
   // console.log('ymlStr', ymlStr)
   return ymlStr

package/src/utils/chalk.js

@@ -0,0 +1,9 @@
+const safeChalk = require('safe-chalk')
+const minimist = require('minimist')
+const argv = minimist(process.argv.slice(2))
+ 
+// If --json flag disable chalk colors
+const DISABLE_COLORS = argv.json || process.env.NO_COLORS
+ 
+// Export chalk instance for usage in CLI
+module.exports = safeChalk(DISABLE_COLORS)

package/src/utils/find-nested-variables.js

@@ -51,6 +51,7 @@ function findNestedVariables(input, regex, variablesKnownTypes, location, debug
     
     // Store match details
     const matchInfo = {
+      varType: undefined,
       location,
       value: input,
       fullMatch: match[0],
@@ -91,32 +92,33 @@ function findNestedVariables(input, regex, variablesKnownTypes, location, debug
     matches[i].varString = matches[i].variable
     /* Save additional meta data about the variable */
     // console.log('matches[i].varString', matches[i].varString)
-    if (variablesKnownTypes && variablesKnownTypes.test(matches[i].varString)) {
-      matches[i].varType = matches[i].varString.match(variablesKnownTypes)[1]
-      if (FALLBACK_REGEX.test(matches[i].varString)) {
-        const split = splitByComma(matches[i].varString, regex)
-        matches[i].hasFallback = true
 
-        matches[i].valueBeforeFallback = split[0]
-        // remove first element from split
-        matches[i].fallbackValues = split.slice(1).map((item) => {
-          // console.log('item', item)
-          const isVariable = variablesKnownTypes.test(item) || VAR_MATCH_REGEX.test(item)
-          const fallbackData = {
-            isVariable,
-            fullMatch: item,
-            variable: item
-          }
+    // if (variablesKnownTypes && variablesKnownTypes.test(matches[i].varString)) {
+    //   matches[i].varType = matches[i].varString.match(variablesKnownTypes)[1] 
+    //   if (FALLBACK_REGEX.test(matches[i].varString)) {
+    //     const split = splitByComma(matches[i].varString, regex)
+    //     matches[i].hasFallback = true
 
-          if (!isVariable && typeof item === 'string') {
-            fallbackData.stringValue = trimQuotes(item)
-            fallbackData.isFallback = true
-          }
+    //     matches[i].valueBeforeFallback = split[0]
+    //     // remove first element from split
+    //     matches[i].fallbackValues = split.slice(1).map((item) => {
+    //       // console.log('item', item)
+    //       const isVariable = variablesKnownTypes.test(item) || VAR_MATCH_REGEX.test(item)
+    //       const fallbackData = {
+    //         isVariable,
+    //         fullMatch: item,
+    //         variable: item,
+    //       }
+
+    //       if (!isVariable && typeof item === 'string') {
+    //         fallbackData.stringValue = trimQuotes(item)
+    //         fallbackData.isResolvedFallback = true
+    //       }
           
-          return fallbackData
-        })
-      }
-    }
+    //       return fallbackData
+    //     })
+    //   }
+    // }
   }
   
   // Second pass: Reconstruct each variable with original nested syntax
@@ -161,9 +163,102 @@ function findNestedVariables(input, regex, variablesKnownTypes, location, debug
     // Reconstruct with all nested variables
     currentMatch.fullMatch = replaceAllPlaceholders(currentMatch.fullMatch, matches)
     currentMatch.variable = replaceAllPlaceholders(currentMatch.variable, matches)
+  }
 
-  
+
+    // We need to store varString - the variable string with placeholders
+  for (let i = 0; i < matches.length; i++) {
+    matches[i].varString = matches[i].variable
+    /* Save additional meta data about the variable */
+    // console.log('matches[i].varString', matches[i].varString)
+
+    if (variablesKnownTypes && variablesKnownTypes.test(matches[i].varString)) {
+      matches[i].varType = matches[i].varString.match(variablesKnownTypes)[1] 
+      if (FALLBACK_REGEX.test(matches[i].varString)) {
+        const split = splitByComma(matches[i].varString, regex)
+        matches[i].hasFallback = true
+
+        matches[i].valueBeforeFallback = split[0]
+        // remove first element from split
+        matches[i].fallbackValues = split.slice(1).map((item) => {
+          // console.log('item', item)
+          const isVariable = variablesKnownTypes.test(item) || VAR_MATCH_REGEX.test(item)
+          const fallbackData = {
+            isVariable,
+            fullMatch: item,
+            variable: item,
+          }
+
+          if (!isVariable && typeof item === 'string') {
+            fallbackData.stringValue = trimQuotes(item)
+            fallbackData.isResolvedFallback = true
+          }
+          
+          return fallbackData
+        })
+      }
+    } else if (typeof matches[i].varType === 'undefined') {
+      matches[i].varType = 'dot.prop'
+    }
   }
+
+  const finalMatches = matches.map((m) => {
+    delete m.placeholder
+    if (typeof m.varType === 'undefined') {
+      /*
+      {
+        varType: 'dot.prop',
+        location: 'resolvedDomainName',
+        value: '${domainByStage.${opt:stage, ${defaultStage}}}',
+        fullMatch: '${domainByStage.${opt:stage, ${defaultStage}}}',
+        variable: 'domainByStage.${opt:stage, ${defaultStage}}',
+        varString: 'domainByStage.__VAR_1__',
+        resolveOrder: 3,
+        start: 0,
+        end: 26
+      }
+      {
+        varType: 'dot.prop',
+        location: 'resolvedDomainName',
+        value: '${domainByStage.${opt:stage, ${defaultStage}}}',
+        fullMatch: '${defaultStage}',
+        variable: 'defaultStage',
+        varString: 'defaultStage',
+        resolveOrder: 1,
+        start: 29,
+        end: 44
+      }
+      */
+      // console.log('m', m)
+    }
+    if (m.hasFallback) {
+      const combinedFallbacks = m.fallbackValues.reduce((acc, f) => {
+        const child = matches.find((m) => m.variable === f.variable)
+        if (child && child.fallbackValues && child.fallbackValues.length) {
+          const split = splitByComma(child.variable, regex)
+          f.valueBeforeFallback = split[0]
+          f.fallbackValues = child.fallbackValues
+        }
+        return acc
+      }, m.fallbackValues)
+      m.fallbackValues = combinedFallbacks
+    }
+    if (m.varType === 'dot.prop') {
+      // const reversedMatches = matches.reverse()
+      // const test = reversedMatches.reduce((acc, f) => {
+      //   console.log('f', f)
+      //   const child = reversedMatches.find((m) => m.variable === f.variable)
+      //   if (child && child.fallbackValues && child.fallbackValues.length) {
+      //     const split = splitByComma(child.variable, regex)
+      //     f.valueBeforeFallback = split[0]
+      //     f.fallbackValues = child.fallbackValues
+      //   }
+      //   return acc
+      // }, reversedMatches)
+      // console.log('test', test)
+    }
+    return m
+  })
   
   if (debug) {
     console.log("\nReconstructed matches:")
@@ -176,7 +271,7 @@ function findNestedVariables(input, regex, variablesKnownTypes, location, debug
     })
   }
   
-  return matches
+  return finalMatches
 }
 
 

package/src/utils/logs.js

@@ -0,0 +1,15 @@
+const { makeHeader } = require('@davidwells/box-logger')
+
+function logHeader(message) {
+  console.log(makeHeader({
+    text: message, 
+    rightBorder: true, 
+    minWidth: 80, 
+    textStyle: 'normal',
+    borderColor: 'cyanBright',
+  }))
+}
+
+module.exports = {
+  logHeader
+}