configorama 0.6.6 → 0.6.8

package/src/resolvers/valueFromGit.js

@@ -21,11 +21,31 @@ async function _exec(cmd, options = { timeout: 1000 }) {
   })
 }
 
+// TODO denote computed fields in metadata
+/*
+{
+  variables: {
+    repo: {
+      value: '${git:repo}',
+      type: 'string',
+      description: 'The repository owner and name',
+    }
+  },
+  computedVariables : {
+    hash: {
+      value: '${git:sha1}',
+      type: 'string',
+      description: 'The current commit hash',
+    }
+  }
+}
+*/
+
 function createResolver(cwd) {
   async function _getValueFromGit(variableString) {
     const variable = variableString.split(`${GIT_PREFIX}:`)[1]
     let value = null
-    // console.log('variableStringvariableString', variableString)
+    // console.log('createResolver variableString', variableString)
     if (variable.match(/^remote/i)) {
       const hasParams = functionRegex.exec(variableString)
       const remoteName = (hasParams && hasParams[2]) ? formatFunctionArgs(hasParams[2]) : 'origin'

package/src/sync.js

@@ -2,6 +2,7 @@ const path = require('path')
 const fs = require('fs')
 const Configorama = require('./main')
 const getFullPath = require('./utils/getFullFilePath')
+const enrichMetadata = require('./utils/enrichMetadata')
 
 /**
  * Force syncronous invocation of async API
@@ -37,7 +38,7 @@ module.exports = function configoramaSync(varSrcs = []) {
       resolver: resolverFunction
     }
   })
-  return (args) => {
+  return async (args) => {
     const { filePath, settings = {} } = args
     const syncSettings = { sync: true }
     if (customVariableSources && customVariableSources.length) {
@@ -45,7 +46,21 @@ module.exports = function configoramaSync(varSrcs = []) {
     }
     const finalSettings = Object.assign({}, settings, syncSettings)
     const options = finalSettings.options || {}
-    const config = new Configorama(filePath, finalSettings)
-    return config.init(options)
+    const instance = new Configorama(filePath, finalSettings)
+    const result = await instance.init(options)
+
+    if (finalSettings.returnMetadata) {
+      const metadata = instance.collectVariableMetadata()
+
+      // Enrich metadata with resolution tracking data collected during execution
+      const enrichedMetadata = enrichMetadata(metadata, instance.resolutionTracking, instance.variableSyntax)
+
+      return {
+        config: result,
+        metadata: enrichedMetadata
+      }
+    }
+
+    return result
   }
 }

package/src/utils/enrichMetadata.js

@@ -0,0 +1,229 @@
+const { splitCsv } = require('./splitCsv')
+
+/**
+ * Extract file path from a file() or text() reference string
+ * @param {string} propertyString - The property string containing file/text reference
+ * @returns {object|null} Object with filePath, or null if no match
+ */
+function extractFilePath(propertyString) {
+  const fileMatch = propertyString.match(/^\$\{(?:file|text)\((.*?)\)/)
+  if (!fileMatch || !fileMatch[1]) {
+    return null
+  }
+  
+  const fileContent = fileMatch[1].trim()
+  const parts = splitCsv(fileContent)
+  let filePath = parts[0].trim()
+  
+  // Remove quotes if present
+  filePath = filePath.replace(/^['"]|['"]$/g, '')
+  
+  return { filePath }
+}
+
+/**
+ * Normalize a file path (add ./ prefix, fix .//, skip deep refs)
+ * @param {string} filePath - The file path to normalize
+ * @returns {string|null} Normalized path, or null if should be skipped
+ */
+function normalizePath(filePath) {
+  // Skip deep references
+  if (filePath.includes('deep:')) {
+    return null
+  }
+  
+  let normalized = filePath
+  
+  // Add ./ prefix for relative paths
+  if (!filePath.startsWith('./') && 
+      !filePath.startsWith('../') && 
+      !filePath.startsWith('/') && 
+      !filePath.startsWith('~')) {
+    normalized = './' + filePath
+  }
+  
+  // Fix double slashes
+  if (normalized.startsWith('.//')) {
+    normalized = normalized.replace('.//', './')
+  }
+  
+  return normalized
+}
+
+// Enriches variable metadata with resolution tracking data
+/**
+ * @param {object} metadata - The metadata object from collectVariableMetadata
+ * @param {object} resolutionTracking - The resolution tracking data from Configorama instance
+ * @param {RegExp} variableSyntax - The variable syntax regex to detect variables in file paths
+ * @returns {object} Enriched metadata with afterInnerResolution and resolvedFileRefs
+ */
+function enrichMetadata(metadata, resolutionTracking, variableSyntax) {
+  if (!resolutionTracking) {
+    return metadata
+  }
+
+  const varKeys = Object.keys(metadata.variables)
+
+  for (const key of varKeys) {
+    const varInstances = metadata.variables[key]
+
+    for (const varData of varInstances) {
+      const pathKey = varData.path
+      const trackingData = resolutionTracking[pathKey]
+
+      if (trackingData && trackingData.calls && varData.resolveDetails) {
+        // The last call represents the final state (all inner vars resolved)
+        const lastCall = trackingData.calls[trackingData.calls.length - 1]
+
+        // For each resolveDetail, find the matching call and set afterInnerResolution
+        for (let i = 0; i < varData.resolveDetails.length; i++) {
+          const detail = varData.resolveDetails[i]
+          const isOutermost = i === varData.resolveDetails.length - 1
+
+          if (isOutermost) {
+            // For the outermost variable, use the last call's propertyString
+            // This shows the state after all inner variables have been resolved
+            let afterResolution = lastCall.propertyString
+            if (afterResolution.startsWith('${') && afterResolution.endsWith('}')) {
+              afterResolution = afterResolution.slice(2, -1)
+            }
+            detail.afterInnerResolution = afterResolution
+
+            if (lastCall.resolvedValue !== undefined) {
+              detail.resolvedValue = lastCall.resolvedValue
+            }
+          } else {
+            // For inner variables, try to find a matching call
+            for (const call of trackingData.calls) {
+              const callVar = call.variableString
+              const detailVar = detail.variable
+
+              if (callVar === detailVar || callVar.includes(detail.varString)) {
+                let afterResolution = call.propertyString
+                if (afterResolution.startsWith('${') && afterResolution.endsWith('}')) {
+                  afterResolution = afterResolution.slice(2, -1)
+                }
+                detail.afterInnerResolution = afterResolution
+
+                if (call.resolvedValue !== undefined) {
+                  detail.resolvedValue = call.resolvedValue
+                }
+                break
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // Build resolvedFileRefs array from tracking data
+  // Only use the LAST call for each path (final resolved state)
+  const resolvedFileRefs = []
+  const normalizedPaths = new Set()
+  
+  for (const pathKey in resolutionTracking) {
+    const tracking = resolutionTracking[pathKey]
+    if (tracking.calls && tracking.calls.length) {
+      const lastCall = tracking.calls[tracking.calls.length - 1]
+      
+      const extracted = extractFilePath(lastCall.propertyString)
+      if (extracted) {
+        const normalizedPath = normalizePath(extracted.filePath)
+        
+        if (normalizedPath && !normalizedPaths.has(normalizedPath)) {
+          normalizedPaths.add(normalizedPath)
+          resolvedFileRefs.push(normalizedPath)
+        }
+      }
+    }
+  }
+
+  metadata.resolvedFileRefs = resolvedFileRefs
+
+  // Build resolvedFileRefsData array - maps resolved paths to their variable strings and glob patterns
+  const resolvedFileRefsDataMap = new Map()
+  
+  // First pass: collect all resolved paths and their original variable strings
+  for (const pathKey in resolutionTracking) {
+    const tracking = resolutionTracking[pathKey]
+    if (!tracking.calls || !tracking.calls.length) {
+      continue
+    }
+    
+    const lastCall = tracking.calls[tracking.calls.length - 1]
+    const extracted = extractFilePath(lastCall.propertyString)
+    
+    if (!extracted) {
+      continue
+    }
+    
+    const resolvedPath = normalizePath(extracted.filePath)
+    if (!resolvedPath) {
+      continue
+    }
+    
+    // Get original variable string from tracking
+    const originalPropertyString = tracking.originalPropertyString
+    if (!originalPropertyString) {
+      continue
+    }
+    
+    const origExtracted = extractFilePath(originalPropertyString)
+    if (!origExtracted) {
+      continue
+    }
+    
+    const origPath = normalizePath(origExtracted.filePath)
+    if (!origPath) {
+      continue
+    }
+    
+    // Initialize map entry if needed
+    if (!resolvedFileRefsDataMap.has(resolvedPath)) {
+      resolvedFileRefsDataMap.set(resolvedPath, {
+        resolvedPath: resolvedPath,
+        refs: [],
+      })
+    }
+    
+    const entry = resolvedFileRefsDataMap.get(resolvedPath)
+    
+    // Add original variable string with config path if not already present
+    const alreadyExists = entry.refs.some(ref => ref.path === pathKey && ref.value === origPath)
+    if (!alreadyExists) {
+      const refEntry = { path: pathKey, value: origPath }
+      // Check if the value contains variables
+      if (variableSyntax && origPath.match(variableSyntax)) {
+        refEntry.hasVariable = true
+      }
+      entry.refs.push(refEntry)
+    }
+  }
+  
+  // Second pass: generate glob patterns for each resolved path
+  for (const [resolvedPath, data] of resolvedFileRefsDataMap) {
+    const globPatternSet = new Set()
+    
+    for (const ref of data.refs) {
+      // Check if variable path contains variables
+      if (ref.value.match(variableSyntax)) {
+        const globPattern = ref.value.replace(variableSyntax, '*')
+        globPatternSet.add(globPattern)
+      }
+    }
+    const patterns = Array.from(globPatternSet)
+    if (patterns.length > 0) {
+      data.globPatterns = patterns
+    }
+  }
+  
+  // Convert map to array
+  const resolvedFileRefsData = Array.from(resolvedFileRefsDataMap.values())
+  
+  metadata.resolvedFileRefsData = resolvedFileRefsData
+
+  return metadata
+}
+
+module.exports = enrichMetadata

package/src/utils/find-nested-variables.js

@@ -22,6 +22,7 @@ const VAR_MATCH_REGEX = /__VAR_\d+__/
  * @returns {Array} Array of match objects with fullMatch, variable, varString and other properties
  */
 function findNestedVariables(input, regex, variablesKnownTypes, location, debug = false) {
+  // console.log('variablesKnownTypes', variablesKnownTypes)
   // Create a copy of the input for replacement tracking
   let current = input
   // console.log('current', current)
@@ -182,7 +183,9 @@ function findNestedVariables(input, regex, variablesKnownTypes, location, debug
         // remove first element from split
         matches[i].fallbackValues = split.slice(1).map((item) => {
           // console.log('item', item)
-          const isVariable = variablesKnownTypes.test(item) || VAR_MATCH_REGEX.test(item)
+          // Strip ${} wrapper if present to properly test against variablesKnownTypes
+          const innerContent = item.replace(/^\$\{(.*)\}$/, '$1')
+          const isVariable = variablesKnownTypes.test(innerContent) || VAR_MATCH_REGEX.test(item)
           const fallbackData = {
             isVariable,
             fullMatch: item,
@@ -195,7 +198,7 @@ function findNestedVariables(input, regex, variablesKnownTypes, location, debug
           }
 
           if (isVariable) {
-            const varType = item.match(variablesKnownTypes)[1]
+            const varType = innerContent.match(variablesKnownTypes)[1]
             fallbackData.varType = varType
             // if (varType === 'self:') {
             //   fallbackData.fullMatch = item.replace('self:', '')
@@ -290,13 +293,13 @@ function findNestedVariables(input, regex, variablesKnownTypes, location, debug
  * @param {boolean} debug - Whether to print debug information
  * @returns {Array} Array of match objects containing full match and captured group
  */
-function findNestedVariablesx(input, regex, variablesKnownTypes, debug = false) {
+function findNestedVariablesOld(input, regex, variablesKnownTypes, debug = false) {
   let str = input
   let matches = []
   let match
   let iteration = 0
 
-  console.log('input', input)
+  // console.log('input', input)
   
   if (debug) console.log(`Initial string: ${str}`)
 

package/src/utils/find-nested-variables.test.js

@@ -106,12 +106,51 @@ test('findNestedVariables - mutliple fallback items', () => {
   assert.equal(result[result.length - 1].variable, 'file(./config.${opt:stage, ${opt:stageOne}, ${opt:stageTwo}, "three"}.json)');
 });
 
-test.skip('findNestedVariables - deep', () => {
-  const input = '${file(./config.${opt:stage, ${opt:stageOne, ${env:foo}}, ${opt:stageTwo}, "three" }.json)}';
+test('findNestedVariables - deep', () => {
+  const input =
+  '${file(./config.${opt:stage, ${opt:stageOne, ${env:foo}}, ${opt:stageTwo}, "three" }.json)}';
   const result = findNestedVariables(input, regex, variablesKnownTypes, 'xyz');
   deepLog('result', result)
-  // Check varString property for the outermost variable
-  assert.equal(result[result.length - 1].variable, 'file(./config.${opt:stage, ${opt:stageOne}, ${opt:stageTwo}, "three"}.json)');
+
+  // Should have 5 variables total
+  assert.equal(result.length, 5);
+
+  // Check the innermost variable
+  assert.equal(result[0].fullMatch, '${env:foo}');
+  assert.equal(result[0].variable, 'env:foo');
+  assert.equal(result[0].varType, 'env:');
+
+  // Check opt:stageOne with env:foo fallback
+  assert.equal(result[1].fullMatch, '${opt:stageOne, ${env:foo}}');
+  assert.equal(result[1].variable, 'opt:stageOne, ${env:foo}');
+  assert.equal(result[1].varType, 'opt:');
+  assert.equal(result[1].hasFallback, true);
+  assert.equal(result[1].valueBeforeFallback, 'opt:stageOne');
+  assert.equal(result[1].fallbackValues.length, 1);
+  assert.equal(result[1].fallbackValues[0].isVariable, true);
+  assert.equal(result[1].fallbackValues[0].fullMatch, '${env:foo}');
+  assert.equal(result[1].fallbackValues[0].varType, 'env:');
+
+  // Check opt:stageTwo
+  assert.equal(result[2].fullMatch, '${opt:stageTwo}');
+  assert.equal(result[2].variable, 'opt:stageTwo');
+
+  // Check opt:stage with multiple fallbacks
+  assert.equal(result[3].fullMatch, '${opt:stage, ${opt:stageOne, ${env:foo}}, ${opt:stageTwo}, "three" }');
+  assert.equal(result[3].variable, 'opt:stage, ${opt:stageOne, ${env:foo}}, ${opt:stageTwo}, "three"');
+  assert.equal(result[3].hasFallback, true);
+  assert.equal(result[3].valueBeforeFallback, 'opt:stage');
+  assert.equal(result[3].fallbackValues.length, 3);
+  assert.equal(result[3].fallbackValues[0].fullMatch, '${opt:stageOne, ${env:foo}}');
+  assert.equal(result[3].fallbackValues[0].isVariable, true);
+  assert.equal(result[3].fallbackValues[1].fullMatch, '${opt:stageTwo}');
+  assert.equal(result[3].fallbackValues[1].isVariable, true);
+  assert.equal(result[3].fallbackValues[2].fullMatch, '"three"');
+  assert.equal(result[3].fallbackValues[2].isVariable, false);
+  assert.equal(result[3].fallbackValues[2].stringValue, 'three');
+
+  // Check outermost file variable
+  assert.equal(result[4].variable, 'file(./config.${opt:stage, ${opt:stageOne, ${env:foo}}, ${opt:stageTwo}, "three" }.json)');
 });
 
 

package/src/utils/isValidValue.js

@@ -1,7 +1,7 @@
 const isEmpty = require('lodash.isempty')
 
 module.exports = function isValidValue(val) {
-  if (typeof val === 'object' && val.hasOwnProperty('__internal_only_flag')) {
+  if (typeof val === 'object' && (val.hasOwnProperty('__internal_only_flag') || val.hasOwnProperty('__internal_metadata'))) {
     return false
   }
   return val !== null && typeof val !== 'undefined' && !(typeof val === 'object' && isEmpty(val))

package/src/utils/splitByComma.js

@@ -34,11 +34,13 @@ function splitByComma(string, regexPattern) {
   let current = ""
   let inQuote = false
   let quoteChar = ""
-  let bracketDepth = 0  // Includes both () and []
-  
+  let bracketDepth = 0  // Includes (), [], and {}
+  let dollarBraceDepth = 0  // Track ${ ... } depth separately (only when regexPattern is provided)
+
   for (let i = 0; i < protectedString.length; i++) {
     const char = protectedString[i]
-    
+    const prevChar = i > 0 ? protectedString[i-1] : ''
+
     // Handle quotes
     if ((char === "'" || char === '"') && (i === 0 || protectedString[i-1] !== "\\")) {
       if (!inQuote) {
@@ -48,13 +50,35 @@ function splitByComma(string, regexPattern) {
         inQuote = false
       }
     }
-    
-    // Handle parentheses and brackets
-    if ((char === "(" || char === "[") && !inQuote) bracketDepth++
-    if ((char === ")" || char === "]") && !inQuote) bracketDepth--
-    
+
+    // Handle parentheses, brackets, and curly braces
+    if (!inQuote) {
+      if (char === "(" || char === "[") {
+        bracketDepth++
+      } else if (char === ")" || char === "]") {
+        bracketDepth--
+      } else if (regexPattern) {
+        // Only track {} when we have regexPattern (i.e., when protecting variables)
+        // TODO this doesn't support custom variable syntax regexes.
+        if (char === "{" && prevChar === "$") {
+          // Track ${ as a special unit
+          dollarBraceDepth++
+        } else if (char === "{") {
+          // Standalone { (not part of ${)
+          bracketDepth++
+        } else if (char === "}") {
+          // Check if this closes a ${ or a standalone {
+          if (dollarBraceDepth > 0) {
+            dollarBraceDepth--
+          } else if (bracketDepth > 0) {
+            bracketDepth--
+          }
+        }
+      }
+    }
+
     // Process comma
-    if (char === "," && !inQuote && bracketDepth === 0) {
+    if (char === "," && !inQuote && bracketDepth === 0 && dollarBraceDepth === 0) {
       result.push(current.trim())
       current = ""
     } else {

package/src/utils/splitByComma.test.js

@@ -46,7 +46,7 @@ test('splitByComma - should handle input with extra whitespace', () => {
 
 test('splitByComma - should handle mixed scenarios', () => {
   const result = splitByComma("normal, 'quoted, string', function(param1, param2)")
-  console.log('result', result)
+  // console.log('result', result)
   assert.equal(result, ["normal", "'quoted, string'", "function(param1, param2)"])
 })
 
@@ -80,5 +80,50 @@ test('splitByComma - should handle backtick quotes', () => {
   assert.equal(result, ["normal", "`template ${with, commas} inside`"])
 })
 
-// Run all tests  
+test('splitByComma - should split ${} variables when NO regex provided', () => {
+  const result = splitByComma('${env:no, env:empty}')
+  assert.equal(result, ['${env:no', 'env:empty}'])
+})
+
+test('splitByComma - should protect ${} variables when regex provided', () => {
+  const result = splitByComma('opt:stage, ${opt:stageOne}, ${opt:stageTwo}', variableSyntax)
+  assert.equal(result, ['opt:stage', '${opt:stageOne}', '${opt:stageTwo}'])
+})
+
+test('splitByComma - should protect nested ${} variables when regex provided', () => {
+  const result = splitByComma('opt:stage, ${opt:stageOne, ${env:foo}}, ${opt:stageTwo}', variableSyntax)
+  assert.equal(result, ['opt:stage', '${opt:stageOne, ${env:foo}}', '${opt:stageTwo}'])
+})
+
+test('splitByComma - should handle multiple nested levels with regex', () => {
+  const result = splitByComma('${opt:stageOne, ${env:foo}}, ${opt:stageTwo}, "three"', variableSyntax)
+  assert.equal(result, ['${opt:stageOne, ${env:foo}}', '${opt:stageTwo}', '"three"'])
+})
+
+test('splitByComma - should handle standalone {} objects with regex', () => {
+  const result = splitByComma('func(arg1, {key: value}, arg2)', variableSyntax)
+  assert.equal(result, ['func(arg1, {key: value}, arg2)'])
+})
+
+test('splitByComma - should handle mixed ${} variables and {} objects with regex', () => {
+  const result = splitByComma('${self:config}, {key: "value"}, ${env:var}', variableSyntax)
+  assert.equal(result, ['${self:config}', '{key: "value"}', '${env:var}'])
+})
+
+test('splitByComma - should handle complex nested case from failCases', () => {
+  const result = splitByComma('${env:no, env:empty}')
+  assert.equal(result, ['${env:no', 'env:empty}'])
+})
+
+test('splitByComma - should handle deeply nested variables with regex', () => {
+  const input = 'opt:stage, ${opt:stageOne, ${env:foo}}, ${opt:stageTwo}, "three"'
+  const result = splitByComma(input, variableSyntax)
+  assert.equal(result.length, 4)
+  assert.equal(result[0], 'opt:stage')
+  assert.equal(result[1], '${opt:stageOne, ${env:foo}}')
+  assert.equal(result[2], '${opt:stageTwo}')
+  assert.equal(result[3], '"three"')
+})
+
+// Run all tests
 test.run()