configorama 0.8.0 → 0.9.3

package/README.md

@@ -870,14 +870,49 @@ const config = await configorama(configFile, {
   allowUnresolvedVariables: ['param', 'file'],  // only these pass through
   options: { stage: 'prod' }
 })
-// Unresolved ${param:x} and ${file(missing.yml)} pass through
-// Unresolved ${env:MISSING} throws an error
+// Input:  { paramKey: '${param:x}', fileKey: '${file(missing.yml)}' }
+// Output: { paramKey: '${param:x}', fileKey: '${file(missing.yml)}' }
+
+// Allow only SPECIFIC types to be unresolved
+const config = await configorama(configFile, {
+  allowUnresolvedVariables: ['param', 'file'],  // only these pass through
+  options: { stage: 'prod' }
+})
+// Input:  { key: '${env:MISSING_VAR}', paramKey: '${param:x}', fileKey: '${file(missing.yml)}' }
+// Unresolved ${param:x} and ${file(missing.yml)} pass through but
+// Output error thrown because ${env:MISSING_VAR} throws an error
 ```
 
-This is useful for multi-stage resolution (e.g., Serverless Dashboard resolves params after local resolution).
+This is useful for multi-stage resolution (e.g., Downstream Serverless Dashboard resolves params after local resolution).
+
+> **Note:** This option does NOT apply to `self:` or dotProp variables (e.g., `${foo.bar.baz}`). These are local references that configorama fully owns—if they can't be resolved, it's a config error, not something to defer to another system.
 
 ## FAQ
 
+**Q: What happens with circular variable dependencies?**
+
+Configorama detects circular dependencies and throws a helpful error instead of hanging forever.
+
+```yml
+# Direct cycle - throws error
+a: ${self:b}
+b: ${self:a}
+# Error: Circular variable dependency detected: b → a → b
+
+# Indirect cycle - also detected
+a: ${self:b}
+b: ${self:c}
+c: ${self:a}
+# Error: Circular variable dependency detected: c → a → b → c
+
+# Works with shorthand syntax too
+foo:
+  bar: ${baz.qux}
+baz:
+  qux: ${foo.bar}
+# Error: Circular variable dependency detected: baz.qux → foo.bar → baz.qux
+```
+
 **Q: Why should I use this?**
 
 Never rendering a stale configuration file again!

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "configorama",
-  "version": "0.8.0",
+  "version": "0.9.3",
   "description": "Variable support for configuration files",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/main.js

@@ -121,6 +121,7 @@ class Configorama {
       allowUndefinedValues: false,
       // Allow known variable types that can't be resolved to pass through
       // Can be: false | true | ['param', 'file', 'env', ...]
+      // Note: Does not apply to self: or dotprop refs - those always error
       allowUnresolvedVariables: false,
       // Return metadata
       returnMetadata: false,
@@ -334,21 +335,25 @@ class Configorama {
         /** */
         /* its file ref so we need to shift lookup for self in nested files */
         if (valueObject.isFileRef) {
-          const exists = dotProp.get(fullObject, varString)
-          // console.log('fallThroughSelfMatcher exists', exists)
-          if (!exists) {
-            // @ Todo make recursive
-            const deepProperties = [valueObject.path[0]].concat(varString)
-            const dotPropPath = deepProperties.join('.')
-            const deeperExists = dotProp.get(fullObject, dotPropPath)
-            // console.log('fallThroughSelfMatcher deeper', deeperExists)
-            return deeperExists
+          // First check if property exists in the nested file's context (preferred)
+          const nestedPath = [valueObject.path[0]].concat(varString)
+          const nestedDotPath = nestedPath.join('.')
+          if (dotProp.has(fullObject, nestedDotPath)) {
+            // Property exists in nested context - return true to indicate match
+            // (actual value resolution happens in resolver, not here)
+            return true
           }
+          // Fall back to top-level lookup
+          if (dotProp.has(fullObject, varString)) {
+            return true
+          }
+          return false
         }
         // console.log('fallthrough fullObject', fullObject)
         /* is simple ${whatever} reference in same file */
         const startOf = varString.split('.')
-        return fullObject[startOf[0]]
+        // Use has() to properly check existence for falsy values
+        return dotProp.has(fullObject, startOf[0])
       },
       resolver: (varString, options, config, pathValue) => {
         /*
@@ -2767,6 +2772,25 @@ Missing Value ${missingValue} - ${matchedString}
 
     // console.log('getValueFromSrc propertyString', propertyString)
     // console.log(`tracker contains ${variableString}`, this.tracker.contains(variableString))
+
+    // Cycle detection: track dependencies and check for cycles
+    const fromPath = valueObject.path ? valueObject.path.join('.') : null
+    // Extract target path from variableString (e.g., 'self:b' → 'b', 'b.c' → 'b.c')
+    let toPath = variableString
+    if (variableString.startsWith('self:')) {
+      toPath = variableString.slice(5)
+    }
+    // For cycle detection, only track self-references
+    if (fromPath && (variableString.startsWith('self:') || !variableString.includes(':'))) {
+      if (this.tracker.wouldCreateCycle(fromPath, toPath)) {
+        const cyclePath = this.tracker.getCyclePath(fromPath, toPath)
+        return Promise.reject(new Error(
+          `Circular variable dependency detected: ${cyclePath.join(' → ')}`
+        ))
+      }
+      this.tracker.addDependency(fromPath, toPath)
+    }
+
     if (this.tracker.contains(variableString)) {
       // console.log('try to get', variableString)
       return this.tracker.get(variableString, propertyString)
@@ -2873,7 +2897,7 @@ Missing Value ${missingValue} - ${matchedString}
             // Find the most recent call for this variableString
             for (let i = this.resolutionTracking[pathKey].calls.length - 1; i >= 0; i--) {
               if (this.resolutionTracking[pathKey].calls[i].variableString === variableString) {
-                const v = (typeof val === 'object' && val.__internal_only_flag) ? val.value : val
+                const v = (val && typeof val === 'object' && val.__internal_only_flag) ? val.value : val
                 this.resolutionTracking[pathKey].calls[i].resolvedValue = v
                 this.resolutionTracking[pathKey].calls[i].resolverType = resolverType
                 break
@@ -3244,15 +3268,16 @@ Missing Value ${missingValue} - ${matchedString}
 
     /* its file ref so we need to shift lookup for self in nested files */
     if (data.isFileRef) {
-      const dotPropPath = deepProperties.length > 1 ? deepProperties.join('.') : deepProperties[0]
-      const exists = dotProp.get(valueToPopulate, dotPropPath)
-      // console.log('self exists', exists)
-      if (!exists) {
-        // @ Todo make recursive
-        deepProperties = [data.path[0]].concat(deepProperties)
-        // console.log('self fixed deepProperties', deepProperties)
-      }
+      // First check if property exists in the nested file's context (preferred for file refs)
+      const nestedPath = [data.path[0]].concat(deepProperties)
+      const nestedDotPath = nestedPath.join('.')
+      if (dotProp.has(valueToPopulate, nestedDotPath)) {
+        // Property exists in nested context, prefer it over top-level
+        deepProperties = nestedPath
+      }
+      // Otherwise, keep deepProperties as-is to try top-level lookup
     }
+
     return this.getDeeperValue(deepProperties, valueToPopulate).then((res) => {
       /*
       console.log('self getDeeperValue variableString', variableString)
@@ -3426,7 +3451,7 @@ Missing Value ${missingValue} - ${matchedString}
     })
   }
   runFunction(variableString) {
-    console.log('runFunction', variableString)
+    // console.log('runFunction', variableString)
     /* If json object value return it */
     if (variableString.match(/^\s*{/) && variableString.match(/}\s*$/)) {
       return variableString
@@ -3452,7 +3477,14 @@ Missing Value ${missingValue} - ${matchedString}
       // TODO fix how commas + spaces are ned
       const splitter = splitCsv(rawArgs, ', ')
       // console.log('splitter', splitter)
-      argsToPass = formatFunctionArgs(splitter)
+      // Recursively evaluate any nested function calls in arguments
+      const evaluatedArgs = splitter.map((arg) => {
+        if (typeof arg === 'string' && funcRegex.test(arg)) {
+          return this.runFunction(arg)
+        }
+        return arg
+      })
+      argsToPass = formatFunctionArgs(evaluatedArgs)
     }
     // console.log('argsToPass runFunction', argsToPass)
     // TODO check for camelCase version. | toUpperCase messes with function name

package/src/parsers/yaml.js

@@ -132,8 +132,8 @@ function preProcess(ymlStr = '') {
           if (txt.indexOf(`'${nested}'`) > -1) {
             return
           }
-          /* Replace variable wrapped in quotes */
-          fixedText = fixedText.replace(nested, `"${nested}"`)
+          /* Replace ALL occurrences of variable wrapped in quotes */
+          fixedText = fixedText.replaceAll(nested, `"${nested}"`)
         })
         ymlStr = ymlStr.replace(txt, fixedText)
       }
@@ -171,8 +171,8 @@ function preProcess(ymlStr = '') {
             if (txt.indexOf(`'${nested}'`) > -1) {
               return
             }
-            /* Replace variable wrapped in quotes */
-            fixedText = fixedText.replace(nested, `"${nested}"`)
+            /* Replace ALL occurrences of variable wrapped in quotes */
+            fixedText = fixedText.replaceAll(nested, `"${nested}"`)
           })
           ymlStr = ymlStr.replace(txt, fixedText)
         }

package/src/parsers/yaml.test.js

@@ -166,4 +166,56 @@ test('preProcess - should handle empty input', () => {
   assert.equal(result, '')
 })
 
+// ==========================================
+// Duplicate variable tests - Bug: String.replace() only replaces first occurrence
+// ==========================================
+
+test('preProcess - should wrap ALL duplicate variables in quotes within same array', () => {
+  const input = `
+items: [\${var:foo}, \${var:foo}, \${var:foo}]
+`
+  const result = preProcess(input)
+
+  // All three occurrences should be wrapped
+  const wrappedCount = (result.match(/"\$\{var:foo\}"/g) || []).length
+
+  assert.is(wrappedCount, 3, `Expected 3 wrapped occurrences, got ${wrappedCount}. Output: ${result}`)
+})
+
+test('preProcess - should wrap ALL duplicate variables in quotes within same object', () => {
+  const input = `
+config: {stage: \${env:STAGE}, region: \${env:STAGE}}
+`
+  const result = preProcess(input)
+
+  // Both occurrences should be wrapped
+  const wrappedCount = (result.match(/"\$\{env:STAGE\}"/g) || []).length
+
+  assert.is(wrappedCount, 2, `Expected 2 wrapped occurrences, got ${wrappedCount}. Output: ${result}`)
+})
+
+test('preProcess - should wrap duplicate variables mixed with unique variables in array', () => {
+  const input = `
+mixed: [\${env:FOO}, \${env:BAR}, \${env:FOO}]
+`
+  const result = preProcess(input)
+
+  const fooCount = (result.match(/"\$\{env:FOO\}"/g) || []).length
+  const barCount = (result.match(/"\$\{env:BAR\}"/g) || []).length
+
+  assert.is(fooCount, 2, `Expected 2 wrapped FOO occurrences, got ${fooCount}. Output: ${result}`)
+  assert.is(barCount, 1, `Expected 1 wrapped BAR occurrence, got ${barCount}. Output: ${result}`)
+})
+
+test('preProcess - should wrap duplicate variables in nested array structure', () => {
+  const input = `
+nested: [[\${opt:stage}, \${opt:stage}], [\${opt:stage}]]
+`
+  const result = preProcess(input)
+
+  const wrappedCount = (result.match(/"\$\{opt:stage\}"/g) || []).length
+
+  assert.is(wrappedCount, 3, `Expected 3 wrapped occurrences, got ${wrappedCount}. Output: ${result}`)
+})
+
 test.run() 

package/src/resolvers/valueFromFile.js

@@ -253,7 +253,19 @@ ${JSON.stringify(options.context, null, 2)}`,
   if (fileExtension === 'js' || fileExtension === 'cjs') {
     const jsFile = require(fullFilePath)
     const { moduleName } = parseModuleReference(variableString, matchedFileString)
-    const returnValueFunction = moduleName ? jsFile[moduleName] : jsFile
+    // For default export functions with :property syntax, keep the function and use deep properties
+    // For named exports (non-function module), look up the named export
+    let returnValueFunction = jsFile
+    let includeFirstProperty = false
+
+    if (moduleName && typeof jsFile === 'function') {
+      // Default export function with property access - include first property in path
+      returnValueFunction = jsFile
+      includeFirstProperty = true
+    } else if (moduleName) {
+      // Named export - look it up directly
+      returnValueFunction = jsFile[moduleName]
+    }
 
     return processExecutableFile({
       fileModule: jsFile,
@@ -264,7 +276,8 @@ ${JSON.stringify(options.context, null, 2)}`,
       matchedFileString,
       relativePath,
       fileType: 'javascript',
-      getDeeperValue: ctx.getDeeperValue
+      getDeeperValue: ctx.getDeeperValue,
+      includeFirstProperty
     })
   }
 

package/src/utils/PromiseTracker.js

@@ -8,6 +8,8 @@ class PromiseTracker {
   reset() {
     this.promiseList = []
     this.promiseMap = {}
+    // Track which variables depend on which (for cycle detection)
+    this.dependencyGraph = {}
     this.startTime = Date.now()
     this.cursor = 0
   }
@@ -81,6 +83,58 @@ class PromiseTracker {
     promise.waitList += ` ${specifier}`
     return promise
   }
+  // Add a dependency edge: "from" depends on "to"
+  addDependency(from, to) {
+    if (!this.dependencyGraph[from]) {
+      this.dependencyGraph[from] = new Set()
+    }
+    this.dependencyGraph[from].add(to)
+  }
+  // Check if adding dependency from → to would create a cycle
+  wouldCreateCycle(from, to) {
+    // Check if "to" can reach "from" (meaning from → to would close a cycle)
+    const visited = new Set()
+    const stack = [to]
+    while (stack.length > 0) {
+      const current = stack.pop()
+      if (current === from) {
+        return true
+      }
+      if (visited.has(current)) {
+        continue
+      }
+      visited.add(current)
+      const deps = this.dependencyGraph[current]
+      if (deps) {
+        for (const dep of deps) {
+          stack.push(dep)
+        }
+      }
+    }
+    return false
+  }
+  // Get the cycle path for error reporting
+  getCyclePath(from, to) {
+    const path = [from, to]
+    const visited = new Set([from, to])
+    let current = to
+    while (current !== from) {
+      const deps = this.dependencyGraph[current]
+      if (!deps) break
+      let found = false
+      for (const dep of deps) {
+        if (dep === from || !visited.has(dep)) {
+          path.push(dep)
+          visited.add(dep)
+          current = dep
+          found = true
+          break
+        }
+      }
+      if (!found) break
+    }
+    return path
+  }
   getPending() {
     return this.promiseList.filter(p => (p.state === 'pending'))
   }

package/src/utils/encoders/unknown-values.js

@@ -20,7 +20,7 @@ function decodeUnknown(rawValue) {
   let val = rawValue.replace(PASSTHROUGH_PATTERN, '')
   if (x.length) {
     x.forEach(({ match, value }) => {
-      const decodedValue = Buffer.from(value, 'base64').toString('ascii')
+      const decodedValue = Buffer.from(value, 'base64').toString('utf8')
       val = val.replace(match, decodedValue)
     })
   }

package/src/utils/encoders/unknown-values.test.js

@@ -0,0 +1,146 @@
+/**
+ * Tests for unknown-values.js - encoding/decoding passthrough variables
+ * 
+ * These functions handle variables that cannot be resolved by Configorama
+ * but need to be passed through to external systems (like Serverless Dashboard).
+ */
+const { test } = require('uvu')
+const assert = require('uvu/assert')
+const {
+  encodeUnknown,
+  decodeUnknown,
+  findUnknownValues,
+  hasEncodedUnknown,
+  PASSTHROUGH_PATTERN
+} = require('./unknown-values')
+
+// ==========================================
+// encodeUnknown tests
+// ==========================================
+
+test('encodeUnknown - encodes string with passthrough wrapper', () => {
+  const result = encodeUnknown('${param:test}')
+  assert.ok(result.startsWith('>passthrough[_['))
+  assert.ok(result.endsWith(']_]'))
+})
+
+test('encodeUnknown - produces valid base64', () => {
+  const result = encodeUnknown('${param:test}')
+  const base64Part = result.replace('>passthrough[_[', '').replace(']_]', '')
+  // Should not throw
+  const decoded = Buffer.from(base64Part, 'base64').toString('utf8')
+  assert.is(decoded, '${param:test}')
+})
+
+// ==========================================
+// decodeUnknown tests - basic functionality
+// ==========================================
+
+test('decodeUnknown - decodes encoded value back to original', () => {
+  const input = '${param:simpleValue}'
+  const encoded = encodeUnknown(input)
+  const decoded = decodeUnknown(encoded)
+  assert.is(decoded, input)
+})
+
+test('decodeUnknown - handles multiple encoded values', () => {
+  const input1 = '${param:first}'
+  const input2 = '${param:second}'
+  const combined = `prefix ${encodeUnknown(input1)} middle ${encodeUnknown(input2)} suffix`
+  const decoded = decodeUnknown(combined)
+  assert.is(decoded, `prefix ${input1} middle ${input2} suffix`)
+})
+
+test('decodeUnknown - returns original string if no encoded values', () => {
+  const input = 'no encoded values here'
+  const result = decodeUnknown(input)
+  assert.is(result, input)
+})
+
+// ==========================================
+// decodeUnknown tests - UTF-8 support (bug fix)
+// ==========================================
+
+test('encodeUnknown/decodeUnknown - handles UTF-8 accented characters', () => {
+  const input = '${param:café}'
+  const encoded = encodeUnknown(input)
+  const decoded = decodeUnknown(encoded)
+  assert.is(decoded, input, 'UTF-8 accented characters should decode correctly')
+})
+
+test('encodeUnknown/decodeUnknown - handles UTF-8 emoji', () => {
+  const input = '${param:celebration-🎉}'
+  const encoded = encodeUnknown(input)
+  const decoded = decodeUnknown(encoded)
+  assert.is(decoded, input, 'UTF-8 emoji should decode correctly')
+})
+
+test('encodeUnknown/decodeUnknown - handles multi-byte UTF-8 characters (Japanese)', () => {
+  const input = '${param:日本語}'
+  const encoded = encodeUnknown(input)
+  const decoded = decodeUnknown(encoded)
+  assert.is(decoded, input, 'Multi-byte UTF-8 characters should decode correctly')
+})
+
+test('encodeUnknown/decodeUnknown - handles Chinese characters', () => {
+  const input = '${param:中文测试}'
+  const encoded = encodeUnknown(input)
+  const decoded = decodeUnknown(encoded)
+  assert.is(decoded, input, 'Chinese characters should decode correctly')
+})
+
+test('encodeUnknown/decodeUnknown - handles mixed ASCII and UTF-8', () => {
+  const input = '${param:hello-世界-🌍}'
+  const encoded = encodeUnknown(input)
+  const decoded = decodeUnknown(encoded)
+  assert.is(decoded, input, 'Mixed ASCII and UTF-8 should decode correctly')
+})
+
+test('encodeUnknown/decodeUnknown - handles special UTF-8 punctuation', () => {
+  const input = '${param:price-€100-£50}'
+  const encoded = encodeUnknown(input)
+  const decoded = decodeUnknown(encoded)
+  assert.is(decoded, input, 'UTF-8 currency symbols should decode correctly')
+})
+
+// ==========================================
+// findUnknownValues tests
+// ==========================================
+
+test('findUnknownValues - finds encoded values in text', () => {
+  const encoded = encodeUnknown('${param:test}')
+  const results = findUnknownValues(encoded)
+  assert.is(results.length, 1)
+  assert.ok(results[0].match.startsWith('[_['))
+  assert.ok(results[0].value.length > 0)
+})
+
+test('findUnknownValues - returns empty array for text without encoded values', () => {
+  const results = findUnknownValues('no encoded values')
+  assert.is(results.length, 0)
+})
+
+test('findUnknownValues - finds multiple encoded values', () => {
+  const text = `${encodeUnknown('${a}')} and ${encodeUnknown('${b}')}`
+  const results = findUnknownValues(text)
+  assert.is(results.length, 2)
+})
+
+// ==========================================
+// hasEncodedUnknown tests
+// ==========================================
+
+test('hasEncodedUnknown - returns true for encoded values', () => {
+  const encoded = encodeUnknown('${param:test}')
+  assert.is(hasEncodedUnknown(encoded), true)
+})
+
+test('hasEncodedUnknown - returns false for plain text', () => {
+  assert.is(hasEncodedUnknown('plain text'), false)
+})
+
+test('hasEncodedUnknown - returns false for regular variables', () => {
+  assert.is(hasEncodedUnknown('${param:test}'), false)
+})
+
+test.run()

package/src/utils/lodash.js

@@ -32,14 +32,15 @@ function set(object, path, value) {
   const lastIndex = keys.length - 1;
   
   for (let i = 0; i < lastIndex; i++) {
-    const key = keys[i];
+    const key = keys[i]
     
-    if (current[key] === undefined) {
+    // Check if value is undefined, null, or not an object (primitives can't have properties)
+    if (current[key] == null || typeof current[key] !== 'object') {
       // Create appropriate container based on next key type
-      current[key] = Number.isInteger(keys[i + 1]) && keys[i + 1] >= 0 ? [] : {};
+      current[key] = Number.isInteger(keys[i + 1]) && keys[i + 1] >= 0 ? [] : {}
     }
     
-    current = current[key];
+    current = current[key]
   }
   
   current[keys[lastIndex]] = value;