configorama 0.6.12 → 0.6.13

package/src/utils/{enrichMetadata.js → parsing/enrichMetadata.js}

@@ -1,7 +1,8 @@
 const dotProp = require('dot-prop')
 const fs = require('fs')
 const path = require('path')
-const { normalizePath, extractFilePath, normalizeFileVariable, resolveInnerVariables } = require('./filePathUtils')
+const { normalizePath, extractFilePath, normalizeFileVariable, resolveInnerVariables } = require('../paths/filePathUtils')
+const { preResolveString, preResolveSingle } = require('../resolution/preResolveVariable')
 
 // Type filters that indicate expected value types
 const TYPE_FILTERS = ['Boolean', 'String', 'Number', 'Array', 'Object', 'Json']
@@ -87,6 +88,18 @@ function createOccurrence(instance, varMatch, options = {}) {
   return occurrence
 }
 
+/**
+ * Get source type for a variable type
+ * @param {string} variableType - The variable type (e.g., 'env', 'opt', 'git')
+ * @param {Array} variableTypes - Array of variable type definitions
+ * @returns {string|undefined} The source type ('user', 'config', 'readonly', 'remote')
+ */
+function getSourceForType(variableType, variableTypes) {
+  if (!variableTypes || !variableType) return undefined
+  const typeDef = variableTypes.find(vt => vt.type === variableType)
+  return typeDef?.source
+}
+
 /**
  * Enriches variable metadata with resolution tracking data.
  * @param {object} metadata - The metadata object from collectVariableMetadata.
@@ -96,21 +109,31 @@ function createOccurrence(instance, varMatch, options = {}) {
  * @param {object} originalConfig - The original config object (before resolution) for self/dot.prop lookups.
  * @param {string} configPath - The path to the config file.
  * @param {Array} filterNames - Array of known filter names.
- * @returns {object} Enriched metadata with resolution details and a complete file reference list.
+ * @param {object} [resolvedConfig] - The resolved config object (optional, for post-resolution enrichment).
+ * @param {object} [options] - CLI options object for opt: resolution.
+ * @param {Array} [variableTypes] - Array of variable type definitions with source info.
+ * @returns {Promise<object>} Enriched metadata with resolution details and a complete file reference list.
  */
-function enrichMetadata(
+async function enrichMetadata(
   metadata,
   resolutionTracking,
   variableSyntax,
   fileRefsFound = [],
   originalConfig = {},
   configPath,
-  filterNames = []
+  filterNames = [],
+  resolvedConfig,
+  options = {},
+  variableTypes = []
 ) {
   if (!resolutionTracking) {
     return metadata
   }
 
+  // Create resolve context early for resolving descriptions
+  const configDir = configPath ? path.dirname(configPath) : undefined
+  const resolveContext = { config: originalConfig, variableSyntax, configDir, options }
+
   const varKeys = Object.keys(metadata.variables)
 
   for (const key of varKeys) {
@@ -124,6 +147,11 @@ function enrichMetadata(
         // For each resolveDetail, find the matching resolution history entry
         for (let i = 0; i < varData.resolveDetails.length; i++) {
           const detail = varData.resolveDetails[i]
+          // Add source type for this variable type
+          const sourceType = getSourceForType(detail.variableType, variableTypes)
+          if (sourceType) {
+            detail.variableSourceType = sourceType
+          }
           const isOutermost = i === varData.resolveDetails.length - 1
 
           if (isOutermost && trackingData.resolutionHistory.length > 0) {
@@ -167,7 +195,8 @@ function enrichMetadata(
         varData.type = instanceType
       }
       if (instanceDescription) {
-        varData.description = instanceDescription
+        // Resolve any variables in the description (e.g., ${allowedValues} -> actual values)
+        varData.description = await preResolveString(instanceDescription, resolveContext)
       }
     }
   }
@@ -317,11 +346,32 @@ function enrichMetadata(
   for (const key of varKeys) {
     const varInstances = metadata.variables[key]
     const firstInstance = varInstances[0]
-    const lastResolveDetail = firstInstance.resolveDetails[firstInstance.resolveDetails.length - 1]
+
+    // Extract variable name from key (e.g. "${self:service}" -> "self:service")
+    const keyVarName = key.slice(2, -1).split(',')[0].trim()
+
+    // Find the resolveDetail that matches THIS key's variable (not just outermost)
+    let matchingDetail = null
+    for (const instance of varInstances) {
+      if (instance.resolveDetails && instance.resolveDetails.length > 0) {
+        const found = instance.resolveDetails.find((detail) => {
+          const detailVar = detail.valueBeforeFallback || detail.variable
+          return detailVar === keyVarName
+        })
+        if (found) {
+          matchingDetail = found
+          break
+        }
+      }
+    }
+    // Fallback to last resolveDetail if no match found
+    if (!matchingDetail) {
+      matchingDetail = firstInstance.resolveDetails[firstInstance.resolveDetails.length - 1]
+    }
 
     // Get the base variable name without fallback
     // Use valueBeforeFallback if present, otherwise use the variable string
-    let baseVar = lastResolveDetail.valueBeforeFallback || lastResolveDetail.variable
+    let baseVar = matchingDetail.valueBeforeFallback || matchingDetail.variable
 
     // Strip filters from baseVar using known filter names
     // e.g., "opt:stage | toUpperCase | help(...)" -> "opt:stage"
@@ -344,7 +394,8 @@ function enrichMetadata(
     if (!uniqueVariablesMap.has(baseVar)) {
       uniqueVariablesMap.set(baseVar, {
         variable: baseVar,
-        variableType: lastResolveDetail.variableType,
+        variableType: matchingDetail.variableType,
+        variableSourceType: getSourceForType(matchingDetail.variableType, variableTypes),
         occurrences: [],
         innerVariables: [],
       })
@@ -365,9 +416,25 @@ function enrichMetadata(
         // The outermost variable is the last one in resolveDetails
         const outermostDetail = instance.resolveDetails[instance.resolveDetails.length - 1]
 
+        // Find position of first | (filter separator) in the outermost variable's original string
+        // Variables after this position are filter-internal (e.g., inside help('...${var}...'))
+        const originalStr = outermostDetail.originalStringValue || ''
+        const outerVarStart = outermostDetail.start || 0
+        // Find first | that's inside the outermost variable (after its opening ${)
+        const filterSeparatorPos = originalStr.indexOf('|', outerVarStart + 2)
+
         for (let i = 0; i < instance.resolveDetails.length - 1; i++) {
           const detail = instance.resolveDetails[i]
 
+          // Check if this variable is inside filter arguments (after the first |)
+          // These are meta-variables used to build filter values, not user-configurable variables
+          const isInsideFilter = filterSeparatorPos !== -1 && detail.start > filterSeparatorPos
+          if (isInsideFilter) {
+            // Mark this detail as filter-internal and skip adding to uniqueVariables
+            detail.isFilterInnerVariable = true
+            continue
+          }
+
           // Check if this variable is actually INSIDE the outermost variable's boundaries
           // A variable is "inner" only if it's contained within the parent's start/end range
           const isInnerVariable = detail.start >= outermostDetail.start && detail.end <= outermostDetail.end
@@ -385,6 +452,7 @@ function enrichMetadata(
               uniqueVariablesMap.set(normalizedSiblingVar, {
                 variable: normalizedSiblingVar,
                 variableType: detail.variableType,
+                variableSourceType: getSourceForType(detail.variableType, variableTypes),
                 occurrences: [],
                 innerVariables: [],
               })
@@ -393,11 +461,29 @@ function enrichMetadata(
             const siblingEntry = uniqueVariablesMap.get(normalizedSiblingVar)
 
             // Add occurrence for this sibling variable
+            // Check if this self-reference resolves to a value in originalConfig
+            let siblingDefaultValue = detail.hasFallback ? (detail.fallbackValues?.[0]?.stringValue || detail.fallbackValues?.[0]?.variable) : undefined
+            let siblingDefaultValueSrc
+            let siblingIsRequired = !detail.hasFallback
+
+            if (detail.variableType === 'self' || detail.variableType === 'dot.prop') {
+              const varPath = (detail.valueBeforeFallback || detail.variable).replace('self:', '')
+              const resolvedValue = dotProp.get(originalConfig, varPath)
+              if (resolvedValue !== undefined) {
+                siblingDefaultValue = resolvedValue
+                siblingDefaultValueSrc = varPath
+                siblingIsRequired = false
+              }
+            }
+
             const siblingOccurrence = createOccurrence(instance, detail.varMatch, {
-              isRequired: !detail.hasFallback,
+              isRequired: siblingIsRequired,
               hasFallback: !!detail.hasFallback,
-              defaultValue: detail.hasFallback ? (detail.fallbackValues?.[0]?.stringValue || detail.fallbackValues?.[0]?.variable) : undefined,
+              defaultValue: siblingDefaultValue,
             })
+            if (siblingDefaultValueSrc) {
+              siblingOccurrence.defaultValueSrc = siblingDefaultValueSrc
+            }
 
             // Check if this exact occurrence already exists
             const occurrenceExists = siblingEntry.occurrences.some(occ =>
@@ -547,7 +633,7 @@ function enrichMetadata(
   }
 
   // Aggregate types and descriptions for each uniqueVariable
-  for (const [, entry] of uniqueVariablesMap) {
+  for (const [varKey, entry] of uniqueVariablesMap) {
     // Collect unique types from occurrences
     const types = entry.occurrences
       .map(occ => occ.type)
@@ -556,18 +642,94 @@ function enrichMetadata(
       entry.types = types
     }
 
-    // Collect unique descriptions from occurrences
-    const descriptions = entry.occurrences
+    // Collect unique descriptions from occurrences and resolve any variables
+    const rawDescriptions = entry.occurrences
       .map(occ => occ.description)
       .filter((d, i, a) => d && a.indexOf(d) === i)
-    if (descriptions.length > 0) {
-      entry.descriptions = descriptions
+
+    if (rawDescriptions.length > 0) {
+      // Build map from raw -> resolved description
+      const descriptionMap = new Map()
+      await Promise.all(
+        rawDescriptions.map(async desc => {
+          // Resolve any variables in the description
+          const resolved = await preResolveString(desc, resolveContext)
+          descriptionMap.set(desc, resolved)
+        })
+      )
+
+      // Update each occurrence's description with resolved version
+      for (const occ of entry.occurrences) {
+        if (occ.description && descriptionMap.has(occ.description)) {
+          occ.description = descriptionMap.get(occ.description)
+        }
+      }
+
+      entry.descriptions = Array.from(descriptionMap.values())
+    }
+
+    // Try to resolve defaultValue for variables that can be pre-resolved
+    // This applies to git:, env:, self: variables without existing defaults
+    const firstOcc = entry.occurrences[0]
+    if (firstOcc && firstOcc.isRequired && !firstOcc.defaultValue) {
+      const resolvableTypes = ['git', 'env', 'self', 'dot.prop']
+      if (resolvableTypes.includes(entry.variableType)) {
+        try {
+          const resolved = await preResolveSingle(entry.variable, resolveContext)
+          if (resolved !== undefined) {
+            const formattedValue = Array.isArray(resolved)
+              ? resolved.join(', ')
+              : (typeof resolved === 'object' ? JSON.stringify(resolved) : String(resolved))
+
+            // Update occurrences and entry
+            entry.occurrences.forEach(occ => {
+              occ.defaultValue = formattedValue
+              occ.isRequired = false
+            })
+            entry.resolvedValue = formattedValue
+          }
+        } catch (e) {
+          // Couldn't resolve, leave as required
+        }
+      }
     }
   }
 
   // Convert map to object for metadata
   metadata.uniqueVariables = Object.fromEntries(uniqueVariablesMap)
 
+  // Add resolvedPropertyValue to resolutionTracking if resolvedConfig provided
+  if (resolvedConfig) {
+    metadata.resolutionHistory = {}
+    for (const pathKey in resolutionTracking) {
+      const tracking = resolutionTracking[pathKey]
+      const keys = pathKey.split('.')
+      let resolvedValue = resolvedConfig
+
+      for (const key of keys) {
+        if (resolvedValue && typeof resolvedValue === 'object') {
+          resolvedValue = resolvedValue[key]
+        } else {
+          resolvedValue = undefined
+          break
+        }
+      }
+
+      // Unescape any __CONFIGVAR:...__ placeholders in tracking data
+      // (resolutionTracking uses escaped config during resolution)
+      const cleanTracking = JSON.parse(
+        JSON.stringify(tracking).replace(/__CONFIGVAR:([A-Za-z0-9+/=]+)__/g, (_, encoded) => {
+          return Buffer.from(encoded, 'base64').toString('utf8')
+        })
+      )
+
+      metadata.resolutionHistory[pathKey] = {
+        ...cleanTracking,
+        resolvedPropertyValue: resolvedValue
+      }
+    }
+  }
+
   return metadata
 }
 

package/src/utils/{parse.js → parsing/parse.js}

@@ -1,9 +1,9 @@
-const YAML = require('../parsers/yaml')
-const TOML = require('../parsers/toml')
-const INI = require('../parsers/ini')
-const JSON5 = require('../parsers/json5')
-const { executeTypeScriptFileSync } = require('../parsers/typescript')
-const { executeESMFileSync } = require('../parsers/esm')
+const YAML = require('../../parsers/yaml')
+const TOML = require('../../parsers/toml')
+const INI = require('../../parsers/ini')
+const JSON5 = require('../../parsers/json5')
+const { executeTypeScriptFileSync } = require('../../parsers/typescript')
+const { executeESMFileSync } = require('../../parsers/esm')
 const cloudFormationSchema = require('./cloudformationSchema')
 
 /**
@@ -18,7 +18,7 @@ const cloudFormationSchema = require('./cloudformationSchema')
 function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}) {
   let configObject
 
-  if (fileType.match(/\.(yml|yaml)/)) {
+  if (fileType.match(/\.(yml|yaml)/i)) {
     try {
       const ymlText = YAML.preProcess(fileContents, varRegex)
       configObject = YAML.parse(ymlText)
@@ -36,14 +36,14 @@ function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}
         configObject = result.data
       }
     }
-  } else if (fileType.match(/\.(toml|tml)/)) {
+  } else if (fileType.match(/\.(toml|tml)/i)) {
     configObject = TOML.parse(fileContents)
-  } else if (fileType.match(/\.(ini)/)) {
+  } else if (fileType.match(/\.(ini)/i)) {
     configObject = INI.parse(fileContents)
-  } else if (fileType.match(/\.(json|json5)/)) {
+  } else if (fileType.match(/\.(json|json5)/i)) {
     configObject = JSON5.parse(fileContents)
   // TODO detect js syntax and use appropriate parser
-  } else if (fileType.match(/\.(js|cjs)/)) {
+  } else if (fileType.match(/\.(js|cjs)/i)) {
     let jsFile
     try {
       jsFile = require(filePath)
@@ -60,7 +60,7 @@ function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}
     } catch (err) {
       throw new Error(err)
     }
-  } else if (fileType.match(/\.(ts|tsx)/)) {
+  } else if (fileType.match(/\.(ts|tsx|mts|cts)/i)) {
     try {
       let jsArgs = opts.dynamicArgs || {}
       if (jsArgs && typeof jsArgs === 'function') {
@@ -76,7 +76,7 @@ function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}
     } catch (err) {
       throw new Error(`Failed to execute TypeScript file ${filePath}: ${err.message}`)
     }
-  } else if (fileType.match(/\.(mjs|esm)/)) {
+  } else if (fileType.match(/\.(mjs|esm)/i)) {
     try {
       let jsArgs = opts.dynamicArgs || {}
       if (jsArgs && typeof jsArgs === 'function') {

package/src/utils/parsing/preProcess.js

@@ -0,0 +1,165 @@
+/**
+ * Preprocesses config to fix malformed fallback references
+ * and escape variables inside help() filter arguments
+ */
+const { splitByComma } = require('../strings/splitByComma')
+
+/**
+ * Preprocess config to fix malformed fallback references
+ * @param {Object} configObject - The parsed configuration object
+ * @param {RegExp} variableSyntax - The variable syntax regex to use
+ * @returns {Object} The preprocessed configuration object
+ */
+function preProcess(configObject, variableSyntax) {
+  // Known reference prefixes that should be wrapped in ${}
+  const refPrefixes = ['self:', 'opt:', 'env:', 'file:', 'text:', 'deep:']
+
+  /**
+   * Escape variables inside help() filter arguments so main resolver skips them
+   * Uses base64 encoding to preserve exact original syntax (supports custom variable syntax)
+   * @param {string} str - String potentially containing help() with variables
+   * @returns {string} String with help() variables escaped
+   */
+  function escapeHelpVariables(str) {
+    if (typeof str !== 'string') return str
+    if (!variableSyntax) return str
+
+    // Match help('...') or help("...") containing variables
+    const helpPattern = /help\(['"]([^'"]+)['"]\)/g
+
+    return str.replace(helpPattern, (match, helpContent) => {
+      // Check if help content contains variables
+      if (!helpContent.match(variableSyntax)) return match
+
+      // Replace each variable match with base64-encoded placeholder
+      const escaped = helpContent.replace(variableSyntax, (varMatch) => {
+        const encoded = Buffer.from(varMatch).toString('base64')
+        return `__CONFIGVAR:${encoded}__`
+      })
+      return `help('${escaped}')`
+    })
+  }
+
+  /**
+   * Fix malformed fallback references in a string
+   * @param {string} str - String potentially containing variables
+   * @returns {string} String with fixed fallback references
+   */
+  function fixFallbacksInString(str) {
+    if (typeof str !== 'string') return str
+
+    let result = str
+    let changed = true
+
+    // Keep iterating until no more changes (to handle nested variables)
+    while (changed) {
+      changed = false
+
+      // Find innermost ${...} blocks (ones that don't contain other ${)
+      let i = 0
+      while (i < result.length) {
+        if (result[i] === '$' && result[i + 1] === '{') {
+          const start = i
+          let braceCount = 1
+          let j = i + 2
+
+          // Find the matching closing brace by counting { and }
+          while (j < result.length && braceCount > 0) {
+            if (result[j] === '{') {
+              braceCount++
+            } else if (result[j] === '}') {
+              braceCount--
+            }
+            j++
+          }
+
+          if (braceCount === 0) {
+            const end = j
+            const match = result.substring(start, end)
+            const content = result.substring(start + 2, end - 1)
+
+            // Only process if there's a comma (indicating fallback syntax)
+            if (content.includes(',')) {
+              // Split by comma
+              const parts = splitByComma(content, variableSyntax)
+
+              if (parts.length > 1) {
+                // Check if the first part has nested ${} - if so, skip this (process inner ones first)
+                const firstPart = parts[0]
+                if (firstPart.includes('${')) {
+                  i = start + 2 // Move past ${ to find inner variables
+                  continue
+                }
+
+                // Check each part after the first (these are fallback values)
+                const fixed = parts.map((part, index) => {
+                  if (index === 0) {
+                    return part // Keep the main reference as-is
+                  }
+
+                  const trimmed = part.trim()
+
+                  // Check if this looks like a reference but is not wrapped
+                  const looksLikeRef = refPrefixes.some(prefix => trimmed.startsWith(prefix))
+                  const alreadyWrapped = trimmed.startsWith('${') && trimmed.endsWith('}')
+
+                  if (looksLikeRef && !alreadyWrapped) {
+                    return ` \${${trimmed}}`
+                  }
+
+                  return ` ${trimmed}`
+                })
+
+                const replacement = `\${${fixed.join(',')}}`
+                if (replacement !== match) {
+                  result = result.substring(0, start) + replacement + result.substring(end)
+                  changed = true
+                  break // Restart search from beginning
+                }
+              }
+            }
+
+            i = start + 2 // Move past ${ to continue searching for nested variables
+          } else {
+            i++
+          }
+        } else {
+          i++
+        }
+      }
+    }
+
+    return result
+  }
+
+  /**
+   * Recursively traverse and fix the config object
+   */
+  function traverseAndFix(obj) {
+    if (typeof obj === 'string') {
+      // First escape help() variables, then fix fallbacks
+      const withHelpEscaped = escapeHelpVariables(obj)
+      return fixFallbacksInString(withHelpEscaped)
+    }
+
+    if (Array.isArray(obj)) {
+      return obj.map(item => traverseAndFix(item))
+    }
+
+    if (obj !== null && typeof obj === 'object') {
+      const result = {}
+      for (const key in obj) {
+        if (obj.hasOwnProperty(key)) {
+          result[key] = traverseAndFix(obj[key])
+        }
+      }
+      return result
+    }
+
+    return obj
+  }
+
+  return traverseAndFix(configObject)
+}
+
+module.exports = preProcess

package/src/utils/{filePathUtils.js → paths/filePathUtils.js}

@@ -1,6 +1,6 @@
 // Utilities for parsing and normalizing file paths in variable references
 
-const { splitCsv } = require('./splitCsv')
+const { splitCsv } = require('../strings/splitCsv')
 
 /**
  * Normalize a file path (add ./ prefix, fix .//, skip deep refs)
@@ -45,12 +45,13 @@ function extractFilePath(variableString) {
     return null
   }
 
+  const { trimSurroundingQuotes } = require('../strings/quoteUtils')
   const fileContent = fileMatch[1].trim()
   const parts = splitCsv(fileContent)
   let filePath = parts[0].trim()
 
   // Remove quotes if present
-  filePath = filePath.replace(/^['"]|['"]$/g, '')
+  filePath = trimSurroundingQuotes(filePath, false)
 
   return { filePath }
 }

package/src/utils/paths/findLineForKey.js

@@ -0,0 +1,47 @@
+/**
+ * Finds the line number for a given key in config file contents
+ */
+
+/**
+ * Find the line number where a key is defined in config file contents
+ * @param {string} keyToFind - The key to search for
+ * @param {string[]} lines - Array of file lines
+ * @param {string} fileType - File extension (e.g., '.yml', '.json', '.toml')
+ * @returns {number} Line number (1-indexed) or 0 if not found
+ */
+function findLineForKey(keyToFind, lines, fileType) {
+  if (!keyToFind || !lines || !lines.length) return 0
+
+  const escapedKey = keyToFind.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+  const lineIdx = lines.findIndex((line) => {
+    // YAML: key: or key :
+    if (fileType === '.yml' || fileType === '.yaml') {
+      return new RegExp(`^\\s*${escapedKey}\\s*:`).test(line)
+    }
+    // TOML: key = or key=
+    if (fileType === '.toml') {
+      return new RegExp(`^\\s*${escapedKey}\\s*=`).test(line)
+    }
+    // JSON: "key": or "key" :
+    if (fileType === '.json' || fileType === '.json5') {
+      return new RegExp(`"${escapedKey}"\\s*:`).test(line)
+    }
+    // INI: key = or key=
+    if (fileType === '.ini') {
+      return new RegExp(`^\\s*${escapedKey}\\s*=`).test(line)
+    }
+    // JS/TS/ESM: key: or "key": or 'key': or `key`: or [`key`]:
+    if (['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts'].includes(fileType)) {
+      return new RegExp(`(?:${escapedKey}|"${escapedKey}"|'${escapedKey}'|\`${escapedKey}\`|\\[\`${escapedKey}\`\\])\\s*:`).test(line)
+    }
+    // Default fallback: try YAML-style
+    return line.includes(`${keyToFind}:`)
+  })
+
+  return lineIdx !== -1 ? lineIdx + 1 : 0
+}
+
+module.exports = {
+  findLineForKey
+}