configorama 0.6.11 → 0.6.13

package/src/utils/{enrichMetadata.js → parsing/enrichMetadata.js}

@@ -1,7 +1,37 @@
-const { splitCsv } = require('./splitCsv')
 const dotProp = require('dot-prop')
 const fs = require('fs')
 const path = require('path')
+const { normalizePath, extractFilePath, normalizeFileVariable, resolveInnerVariables } = require('../paths/filePathUtils')
+const { preResolveString, preResolveSingle } = require('../resolution/preResolveVariable')
+
+// Type filters that indicate expected value types
+const TYPE_FILTERS = ['Boolean', 'String', 'Number', 'Array', 'Object', 'Json']
+
+/**
+ * Extract type filter from filters array
+ * @param {Array} filters - Filters array from variable instance
+ * @returns {string|undefined} Type filter if found
+ */
+function extractTypeFromFilters(filters) {
+  if (!filters || !Array.isArray(filters)) return undefined
+  return filters.find(f => TYPE_FILTERS.includes(f))
+}
+
+/**
+ * Extract description from filters array (help filter)
+ * @param {Array} filters - Filters array from variable instance
+ * @returns {string|undefined} Description if found
+ */
+function extractDescriptionFromFilters(filters) {
+  if (!filters || !Array.isArray(filters)) return undefined
+  for (const filter of filters) {
+    if (filter && typeof filter === 'string') {
+      const helpMatch = filter.match(/^help\(['"](.+)['"]\)$/)
+      if (helpMatch) return helpMatch[1]
+    }
+  }
+  return undefined
+}
 
 /**
  * Create a standardized occurrence object
@@ -29,6 +59,9 @@ function createOccurrence(instance, varMatch, options = {}) {
     }
   }
 
+  // Extract type from filters
+  const type = extractTypeFromFilters(filters)
+
   const occurrence = {
     originalString: instance.originalStringValue,
     varMatch: varMatch,
@@ -40,6 +73,10 @@ function createOccurrence(instance, varMatch, options = {}) {
     hasFallback: options.hasFallback !== undefined ? options.hasFallback : (instance.hasFallback || false),
   }
 
+  if (type) {
+    occurrence.type = type
+  }
+
   if (description) {
     occurrence.description = description
   }
@@ -52,53 +89,15 @@ function createOccurrence(instance, varMatch, options = {}) {
 }
 
 /**
- * Extract file path from a file() or text() reference string
- * @param {string} propertyString - The property string containing file/text reference
- * @returns {object|null} Object with filePath, or null if no match
- */
-function extractFilePath(propertyString) {
-  const fileMatch = propertyString.match(/^\$\{(?:file|text)\((.*?)\)/)
-  if (!fileMatch || !fileMatch[1]) {
-    return null
-  }
-  
-  const fileContent = fileMatch[1].trim()
-  const parts = splitCsv(fileContent)
-  let filePath = parts[0].trim()
-  
-  // Remove quotes if present
-  filePath = filePath.replace(/^['"]|['"]$/g, '')
-  
-  return { filePath }
-}
-
-/**
- * Normalize a file path (add ./ prefix, fix .//, skip deep refs)
- * @param {string} filePath - The file path to normalize
- * @returns {string|null} Normalized path, or null if should be skipped
+ * Get source type for a variable type
+ * @param {string} variableType - The variable type (e.g., 'env', 'opt', 'git')
+ * @param {Array} variableTypes - Array of variable type definitions
+ * @returns {string|undefined} The source type ('user', 'config', 'readonly', 'remote')
  */
-function normalizePath(filePath) {
-  // Skip deep references
-  if (filePath.includes('deep:')) {
-    return null
-  }
-  
-  let normalized = filePath
-  
-  // Add ./ prefix for relative paths
-  if (!filePath.startsWith('./') && 
-      !filePath.startsWith('../') && 
-      !filePath.startsWith('/') && 
-      !filePath.startsWith('~')) {
-    normalized = './' + filePath
-  }
-  
-  // Fix double slashes
-  if (normalized.startsWith('.//')) {
-    normalized = normalized.replace('.//', './')
-  }
-  
-  return normalized
+function getSourceForType(variableType, variableTypes) {
+  if (!variableTypes || !variableType) return undefined
+  const typeDef = variableTypes.find(vt => vt.type === variableType)
+  return typeDef?.source
 }
 
 /**
@@ -110,21 +109,31 @@ function normalizePath(filePath) {
  * @param {object} originalConfig - The original config object (before resolution) for self/dot.prop lookups.
  * @param {string} configPath - The path to the config file.
  * @param {Array} filterNames - Array of known filter names.
- * @returns {object} Enriched metadata with resolution details and a complete file reference list.
+ * @param {object} [resolvedConfig] - The resolved config object (optional, for post-resolution enrichment).
+ * @param {object} [options] - CLI options object for opt: resolution.
+ * @param {Array} [variableTypes] - Array of variable type definitions with source info.
+ * @returns {Promise<object>} Enriched metadata with resolution details and a complete file reference list.
  */
-function enrichMetadata(
+async function enrichMetadata(
   metadata,
   resolutionTracking,
   variableSyntax,
   fileRefsFound = [],
   originalConfig = {},
   configPath,
-  filterNames = []
+  filterNames = [],
+  resolvedConfig,
+  options = {},
+  variableTypes = []
 ) {
   if (!resolutionTracking) {
     return metadata
   }
 
+  // Create resolve context early for resolving descriptions
+  const configDir = configPath ? path.dirname(configPath) : undefined
+  const resolveContext = { config: originalConfig, variableSyntax, configDir, options }
+
   const varKeys = Object.keys(metadata.variables)
 
   for (const key of varKeys) {
@@ -138,6 +147,11 @@ function enrichMetadata(
         // For each resolveDetail, find the matching resolution history entry
         for (let i = 0; i < varData.resolveDetails.length; i++) {
           const detail = varData.resolveDetails[i]
+          // Add source type for this variable type
+          const sourceType = getSourceForType(detail.variableType, variableTypes)
+          if (sourceType) {
+            detail.variableSourceType = sourceType
+          }
           const isOutermost = i === varData.resolveDetails.length - 1
 
           if (isOutermost && trackingData.resolutionHistory.length > 0) {
@@ -173,6 +187,17 @@ function enrichMetadata(
           }
         }
       }
+
+      // Add type and description to individual variable instance
+      const instanceType = extractTypeFromFilters(varData.filters)
+      const instanceDescription = extractDescriptionFromFilters(varData.filters)
+      if (instanceType) {
+        varData.type = instanceType
+      }
+      if (instanceDescription) {
+        // Resolve any variables in the description (e.g., ${allowedValues} -> actual values)
+        varData.description = await preResolveString(instanceDescription, resolveContext)
+      }
     }
   }
 
@@ -198,7 +223,8 @@ function enrichMetadata(
   }
 
   // Update fileDependencies.resolvedPaths with the resolved file refs
-  if (metadata.fileDependencies) {
+  // Only overwrite if we have enriched data, otherwise keep original from collectVariableMetadata
+  if (metadata.fileDependencies && resolvedFileRefs.length > 0) {
     metadata.fileDependencies.resolvedPaths = resolvedFileRefs
   }
 
@@ -304,10 +330,14 @@ function enrichMetadata(
     }
   }
 
-  // Update fileDependencies with the enriched data
+  // Update fileDependencies with the enriched data (only if we have data)
   if (metadata.fileDependencies) {
-    metadata.fileDependencies.byConfigPath = byConfigPath
-    metadata.fileDependencies.references = references
+    if (byConfigPath.length > 0) {
+      metadata.fileDependencies.byConfigPath = byConfigPath
+    }
+    if (references.length > 0) {
+      metadata.fileDependencies.references = references
+    }
   }
 
   // Build uniqueVariables rollup - group by base variable (without fallbacks)
@@ -316,11 +346,32 @@ function enrichMetadata(
   for (const key of varKeys) {
     const varInstances = metadata.variables[key]
     const firstInstance = varInstances[0]
-    const lastResolveDetail = firstInstance.resolveDetails[firstInstance.resolveDetails.length - 1]
+
+    // Extract variable name from key (e.g. "${self:service}" -> "self:service")
+    const keyVarName = key.slice(2, -1).split(',')[0].trim()
+
+    // Find the resolveDetail that matches THIS key's variable (not just outermost)
+    let matchingDetail = null
+    for (const instance of varInstances) {
+      if (instance.resolveDetails && instance.resolveDetails.length > 0) {
+        const found = instance.resolveDetails.find((detail) => {
+          const detailVar = detail.valueBeforeFallback || detail.variable
+          return detailVar === keyVarName
+        })
+        if (found) {
+          matchingDetail = found
+          break
+        }
+      }
+    }
+    // Fallback to last resolveDetail if no match found
+    if (!matchingDetail) {
+      matchingDetail = firstInstance.resolveDetails[firstInstance.resolveDetails.length - 1]
+    }
 
     // Get the base variable name without fallback
     // Use valueBeforeFallback if present, otherwise use the variable string
-    let baseVar = lastResolveDetail.valueBeforeFallback || lastResolveDetail.variable
+    let baseVar = matchingDetail.valueBeforeFallback || matchingDetail.variable
 
     // Strip filters from baseVar using known filter names
     // e.g., "opt:stage | toUpperCase | help(...)" -> "opt:stage"
@@ -338,29 +389,13 @@ function enrichMetadata(
     }
 
     // Normalize file() and text() references
-    if (baseVar.match(/^(?:file|text)\(/)) {
-      // Strip sub-key accessors like :topLevel, :nested.value, etc.
-      baseVar = baseVar.replace(/:[\w.[\]]+$/, '')
-
-      // Normalize path - remove quotes and ensure it starts with ./
-      baseVar = baseVar.replace(/^(file|text)\((.+?)\)/, (match, funcName, filePath) => {
-        // Remove surrounding quotes (single or double)
-        let cleanPath = filePath.trim().replace(/^["']|["']$/g, '')
-
-        // Use normalizePath for consistent normalization (handles ./, .// etc)
-        const normalized = normalizePath(cleanPath)
-        if (normalized) {
-          return `${funcName}(${normalized})`
-        }
-
-        return match
-      })
-    }
+    baseVar = normalizeFileVariable(baseVar)
 
     if (!uniqueVariablesMap.has(baseVar)) {
       uniqueVariablesMap.set(baseVar, {
         variable: baseVar,
-        variableType: lastResolveDetail.variableType,
+        variableType: matchingDetail.variableType,
+        variableSourceType: getSourceForType(matchingDetail.variableType, variableTypes),
         occurrences: [],
         innerVariables: [],
       })
@@ -381,9 +416,25 @@ function enrichMetadata(
         // The outermost variable is the last one in resolveDetails
         const outermostDetail = instance.resolveDetails[instance.resolveDetails.length - 1]
 
+        // Find position of first | (filter separator) in the outermost variable's original string
+        // Variables after this position are filter-internal (e.g., inside help('...${var}...'))
+        const originalStr = outermostDetail.originalStringValue || ''
+        const outerVarStart = outermostDetail.start || 0
+        // Find first | that's inside the outermost variable (after its opening ${)
+        const filterSeparatorPos = originalStr.indexOf('|', outerVarStart + 2)
+
         for (let i = 0; i < instance.resolveDetails.length - 1; i++) {
           const detail = instance.resolveDetails[i]
 
+          // Check if this variable is inside filter arguments (after the first |)
+          // These are meta-variables used to build filter values, not user-configurable variables
+          const isInsideFilter = filterSeparatorPos !== -1 && detail.start > filterSeparatorPos
+          if (isInsideFilter) {
+            // Mark this detail as filter-internal and skip adding to uniqueVariables
+            detail.isFilterInnerVariable = true
+            continue
+          }
+
           // Check if this variable is actually INSIDE the outermost variable's boundaries
           // A variable is "inner" only if it's contained within the parent's start/end range
           const isInnerVariable = detail.start >= outermostDetail.start && detail.end <= outermostDetail.end
@@ -394,23 +445,14 @@ function enrichMetadata(
             const siblingBaseVar = detail.valueBeforeFallback || detail.variable
 
             // Normalize file/text references for sibling too
-            let normalizedSiblingVar = siblingBaseVar
-            if (normalizedSiblingVar.match(/^(?:file|text)\(/)) {
-              // Strip sub-key accessor (e.g., :foo from file(./_inner.yml):foo)
-              normalizedSiblingVar = normalizedSiblingVar.replace(/:[\w.[\]]+$/, '')
-
-              normalizedSiblingVar = normalizedSiblingVar.replace(/^(file|text)\((.+?)\)/, (match, funcName, filePath) => {
-                let cleanPath = filePath.trim().replace(/^["']|["']$/g, '')
-                const normalized = normalizePath(cleanPath)
-                return normalized ? `${funcName}(${normalized})` : match
-              })
-            }
+            const normalizedSiblingVar = normalizeFileVariable(siblingBaseVar)
 
             // Create or get entry for this sibling variable
             if (!uniqueVariablesMap.has(normalizedSiblingVar)) {
               uniqueVariablesMap.set(normalizedSiblingVar, {
                 variable: normalizedSiblingVar,
                 variableType: detail.variableType,
+                variableSourceType: getSourceForType(detail.variableType, variableTypes),
                 occurrences: [],
                 innerVariables: [],
               })
@@ -419,11 +461,29 @@ function enrichMetadata(
             const siblingEntry = uniqueVariablesMap.get(normalizedSiblingVar)
 
             // Add occurrence for this sibling variable
+            // Check if this self-reference resolves to a value in originalConfig
+            let siblingDefaultValue = detail.hasFallback ? (detail.fallbackValues?.[0]?.stringValue || detail.fallbackValues?.[0]?.variable) : undefined
+            let siblingDefaultValueSrc
+            let siblingIsRequired = !detail.hasFallback
+
+            if (detail.variableType === 'self' || detail.variableType === 'dot.prop') {
+              const varPath = (detail.valueBeforeFallback || detail.variable).replace('self:', '')
+              const resolvedValue = dotProp.get(originalConfig, varPath)
+              if (resolvedValue !== undefined) {
+                siblingDefaultValue = resolvedValue
+                siblingDefaultValueSrc = varPath
+                siblingIsRequired = false
+              }
+            }
+
             const siblingOccurrence = createOccurrence(instance, detail.varMatch, {
-              isRequired: !detail.hasFallback,
+              isRequired: siblingIsRequired,
               hasFallback: !!detail.hasFallback,
-              defaultValue: detail.hasFallback ? (detail.fallbackValues?.[0]?.stringValue || detail.fallbackValues?.[0]?.variable) : undefined,
+              defaultValue: siblingDefaultValue,
             })
+            if (siblingDefaultValueSrc) {
+              siblingOccurrence.defaultValueSrc = siblingDefaultValueSrc
+            }
 
             // Check if this exact occurrence already exists
             const occurrenceExists = siblingEntry.occurrences.some(occ =>
@@ -512,12 +572,7 @@ function enrichMetadata(
         }
 
         // Normalize file paths after variable substitution
-        if (resolvedVariable.match(/^(?:file|text)\(/)) {
-          resolvedVariable = resolvedVariable.replace(/^(file|text)\((.+?)\)/, (match, funcName, filePath) => {
-            const normalized = normalizePath(filePath)
-            return normalized ? `${funcName}(${normalized})` : match
-          })
-        }
+        resolvedVariable = normalizeFileVariable(resolvedVariable)
 
         // Update the variable to the resolved version and update map key
         if (resolvedVariable !== baseVar) {
@@ -577,9 +632,104 @@ function enrichMetadata(
     }
   }
 
+  // Aggregate types and descriptions for each uniqueVariable
+  for (const [varKey, entry] of uniqueVariablesMap) {
+    // Collect unique types from occurrences
+    const types = entry.occurrences
+      .map(occ => occ.type)
+      .filter((t, i, a) => t && a.indexOf(t) === i)
+    if (types.length > 0) {
+      entry.types = types
+    }
+
+    // Collect unique descriptions from occurrences and resolve any variables
+    const rawDescriptions = entry.occurrences
+      .map(occ => occ.description)
+      .filter((d, i, a) => d && a.indexOf(d) === i)
+
+    if (rawDescriptions.length > 0) {
+      // Build map from raw -> resolved description
+      const descriptionMap = new Map()
+      await Promise.all(
+        rawDescriptions.map(async desc => {
+          // Resolve any variables in the description
+          const resolved = await preResolveString(desc, resolveContext)
+          descriptionMap.set(desc, resolved)
+        })
+      )
+
+      // Update each occurrence's description with resolved version
+      for (const occ of entry.occurrences) {
+        if (occ.description && descriptionMap.has(occ.description)) {
+          occ.description = descriptionMap.get(occ.description)
+        }
+      }
+
+      entry.descriptions = Array.from(descriptionMap.values())
+    }
+
+    // Try to resolve defaultValue for variables that can be pre-resolved
+    // This applies to git:, env:, self: variables without existing defaults
+    const firstOcc = entry.occurrences[0]
+    if (firstOcc && firstOcc.isRequired && !firstOcc.defaultValue) {
+      const resolvableTypes = ['git', 'env', 'self', 'dot.prop']
+      if (resolvableTypes.includes(entry.variableType)) {
+        try {
+          const resolved = await preResolveSingle(entry.variable, resolveContext)
+          if (resolved !== undefined) {
+            const formattedValue = Array.isArray(resolved)
+              ? resolved.join(', ')
+              : (typeof resolved === 'object' ? JSON.stringify(resolved) : String(resolved))
+
+            // Update occurrences and entry
+            entry.occurrences.forEach(occ => {
+              occ.defaultValue = formattedValue
+              occ.isRequired = false
+            })
+            entry.resolvedValue = formattedValue
+          }
+        } catch (e) {
+          // Couldn't resolve, leave as required
+        }
+      }
+    }
+  }
+
   // Convert map to object for metadata
   metadata.uniqueVariables = Object.fromEntries(uniqueVariablesMap)
 
+  // Add resolvedPropertyValue to resolutionTracking if resolvedConfig provided
+  if (resolvedConfig) {
+    metadata.resolutionHistory = {}
+    for (const pathKey in resolutionTracking) {
+      const tracking = resolutionTracking[pathKey]
+      const keys = pathKey.split('.')
+      let resolvedValue = resolvedConfig
+
+      for (const key of keys) {
+        if (resolvedValue && typeof resolvedValue === 'object') {
+          resolvedValue = resolvedValue[key]
+        } else {
+          resolvedValue = undefined
+          break
+        }
+      }
+
+      // Unescape any __CONFIGVAR:...__ placeholders in tracking data
+      // (resolutionTracking uses escaped config during resolution)
+      const cleanTracking = JSON.parse(
+        JSON.stringify(tracking).replace(/__CONFIGVAR:([A-Za-z0-9+/=]+)__/g, (_, encoded) => {
+          return Buffer.from(encoded, 'base64').toString('utf8')
+        })
+      )
+
+      metadata.resolutionHistory[pathKey] = {
+        ...cleanTracking,
+        resolvedPropertyValue: resolvedValue
+      }
+    }
+  }
+
   return metadata
 }
 

package/src/utils/{parse.js → parsing/parse.js}

@@ -1,9 +1,9 @@
-const YAML = require('../parsers/yaml')
-const TOML = require('../parsers/toml')
-const INI = require('../parsers/ini')
-const JSON5 = require('../parsers/json5')
-const { executeTypeScriptFileSync } = require('../parsers/typescript')
-const { executeESMFileSync } = require('../parsers/esm')
+const YAML = require('../../parsers/yaml')
+const TOML = require('../../parsers/toml')
+const INI = require('../../parsers/ini')
+const JSON5 = require('../../parsers/json5')
+const { executeTypeScriptFileSync } = require('../../parsers/typescript')
+const { executeESMFileSync } = require('../../parsers/esm')
 const cloudFormationSchema = require('./cloudformationSchema')
 
 /**
@@ -18,7 +18,7 @@ const cloudFormationSchema = require('./cloudformationSchema')
 function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}) {
   let configObject
 
-  if (fileType.match(/\.(yml|yaml)/)) {
+  if (fileType.match(/\.(yml|yaml)/i)) {
     try {
       const ymlText = YAML.preProcess(fileContents, varRegex)
       configObject = YAML.parse(ymlText)
@@ -36,14 +36,14 @@ function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}
         configObject = result.data
       }
     }
-  } else if (fileType.match(/\.(toml|tml)/)) {
+  } else if (fileType.match(/\.(toml|tml)/i)) {
     configObject = TOML.parse(fileContents)
-  } else if (fileType.match(/\.(ini)/)) {
+  } else if (fileType.match(/\.(ini)/i)) {
     configObject = INI.parse(fileContents)
-  } else if (fileType.match(/\.(json|json5)/)) {
+  } else if (fileType.match(/\.(json|json5)/i)) {
     configObject = JSON5.parse(fileContents)
   // TODO detect js syntax and use appropriate parser
-  } else if (fileType.match(/\.(js|cjs)/)) {
+  } else if (fileType.match(/\.(js|cjs)/i)) {
     let jsFile
     try {
       jsFile = require(filePath)
@@ -60,7 +60,7 @@ function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}
     } catch (err) {
       throw new Error(err)
     }
-  } else if (fileType.match(/\.(ts|tsx)/)) {
+  } else if (fileType.match(/\.(ts|tsx|mts|cts)/i)) {
     try {
       let jsArgs = opts.dynamicArgs || {}
       if (jsArgs && typeof jsArgs === 'function') {
@@ -76,7 +76,7 @@ function parseFileContents(fileContents, fileType, filePath, varRegex, opts = {}
     } catch (err) {
       throw new Error(`Failed to execute TypeScript file ${filePath}: ${err.message}`)
     }
-  } else if (fileType.match(/\.(mjs|esm)/)) {
+  } else if (fileType.match(/\.(mjs|esm)/i)) {
     try {
       let jsArgs = opts.dynamicArgs || {}
       if (jsArgs && typeof jsArgs === 'function') {