configorama 0.6.12 → 0.6.13

package/src/parsers/yaml.js

@@ -1,6 +1,7 @@
 const YAML = require('js-yaml')
 const TOML = require('./toml')
 const JSON = require('./json5')
+const { findOutermostVariables, findOutermostBracesDepthFirst } = require('../utils/strings/bracketMatcher')
 
 // Loader for custom CF syntax
 function load(contents, options) {
@@ -57,53 +58,8 @@ function toJson(ymlContents) {
   return json
 }
 
-// TODO only works for default var syntax ${}. Maybe fix?
-function findOutermostVariables(text) {
-  let matches = [];
-  let depth = 0;
-  let startIndex = -1;
-
-  for (let i = 0; i < text.length; i++) {
-    if (text[i] === '$' && text[i + 1] === '{') {
-      if (depth === 0) {
-          startIndex = i;
-      }
-      depth++;
-      i++; // Skip '{'
-    } else if (text[i] === '}') {
-      depth--;
-      if (depth === 0 && startIndex !== -1) {
-        matches.push(text.substring(startIndex, i + 1));
-        startIndex = -1;
-      }
-    }
-  }
-  return matches;
-}
-
-
-function matchOutermostBraces(text) {
-  let depth = 0
-  let startIndex = -1
-  let results = []
-
-  for (let i = 0; i < text.length; i++) {
-    if (text[i] === '{') {
-      if (depth === 0) {
-        startIndex = i
-      }
-      depth++
-    } else if (text[i] === '}') {
-      depth--
-      if (depth === 0 && startIndex !== -1) {
-        results.push(text.substring(startIndex, i + 1))
-        startIndex = -1
-      }
-    }
-  }
-
-  return results
-}
+// Alias for backward compatibility
+const matchOutermostBraces = findOutermostBracesDepthFirst
 
 
 // https://regex101.com/r/XIltbc/1

package/src/resolvers/valueFromCron.js

@@ -1,3 +1,4 @@
+const { trimSurroundingQuotes } = require('../utils/strings/quoteUtils')
 const cronRefSyntax = RegExp(/^cron\((~?[\{\}\:\$a-zA-Z0-9._\-\/,'"\*\` ]+?)?\)/g)
 
 /**
@@ -226,7 +227,7 @@ Examples:
   }
   
   // Remove surrounding quotes if present
-  const cleanExpression = cronExpression.replace(/^['"`](.*)['"`]$/, '$1')
+  const cleanExpression = trimSurroundingQuotes(cronExpression, true)
   
   // If already a cron expression, return it
   if (cleanExpression.match(/^[\*\/,\-\d]+$/)) {
@@ -244,6 +245,7 @@ Examples:
 
 module.exports = {
   type: 'cron',
+  source: 'readonly',
   prefix: 'cron',
   syntax: '${cron(expression)}',
   description: 'Resolves cron expressions. Examples: ${cron("every 5 minutes"}, ${cron("weekdays")}, ${cron("at 9:30")}',

package/src/resolvers/valueFromEnv.js

@@ -24,6 +24,7 @@ Example: \${env:MY_ENV_VAR}
 
 module.exports = {
   type: 'env',
+  source: 'user',
   syntax: '${env:ENV_VAR}',
   description: 'Resolves environment variables. Examples: ${env:MY_ENV_VAR}, ${env:MY_ENV_VAR_TWO, "fallbackValue"}',
   match: envRefSyntax,

package/src/resolvers/valueFromEval.js

@@ -32,6 +32,7 @@ async function getValueFromEval(variableString) {
 
 module.exports = {
   type: 'eval',
+  source: 'readonly',
   description: '${eval(expression)} - Evaluates mathematical expressions',
   match: evalRefSyntax,
   resolver: getValueFromEval

package/src/resolvers/valueFromFile.js

@@ -0,0 +1,394 @@
+/**
+ * Resolves values from file references (file() and text() syntax)
+ */
+const fs = require('fs')
+const { trim } = require('../utils/lodash')
+const { splitCsv } = require('../utils/strings/splitCsv')
+const { resolveFilePathFromMatch } = require('../utils/paths/getFullFilePath')
+const { findNestedVariables } = require('../utils/variables/findNestedVariables')
+const { makeBox } = require('@davidwells/box-logger')
+const { encodeJsSyntax } = require('../utils/encoders/js-fixes')
+
+/* File Parsers */
+const YAML = require('../parsers/yaml')
+const TOML = require('../parsers/toml')
+const INI = require('../parsers/ini')
+const JSON5 = require('../parsers/json5')
+
+/**
+ * Parse file contents based on file extension
+ * @param {string} content - Raw file contents
+ * @param {string} filePath - File path (used to determine extension)
+ * @returns {*} Parsed content
+ */
+function parseFileContents(content, filePath) {
+  const ext = filePath.split('.').pop().toLowerCase()
+
+  if (ext === 'json' || ext === 'json5') {
+    return JSON5.parse(content)
+  }
+  if (ext === 'yml' || ext === 'yaml') {
+    return YAML.parse(content)
+  }
+  if (ext === 'toml' || ext === 'tml') {
+    return TOML.parse(content)
+  }
+  if (ext === 'ini') {
+    return INI.parse(content)
+  }
+
+  // Return raw content for other files
+  return content
+}
+
+/**
+ * Resolves a value from a file reference
+ * @param {object} ctx - Context object with instance properties
+ * @param {string} ctx.configPath - Base path for file resolution
+ * @param {Array} ctx.fileRefsFound - Mutable array tracking file refs
+ * @param {RegExp} ctx.variableSyntax - Regex for variable syntax
+ * @param {object} ctx.variablesKnownTypes - Known variable types
+ * @param {object} ctx.variableTypes - Variable types
+ * @param {object} ctx.opts - Options object
+ * @param {object} ctx.originalConfig - Original config
+ * @param {object} ctx.config - Current config
+ * @param {Function} ctx.getDeeperValue - Method for nested lookups
+ * @param {RegExp} ctx.fileRefSyntax - Regex for file() syntax
+ * @param {RegExp} ctx.textRefSyntax - Regex for text() syntax
+ * @param {string} variableString - The variable string to resolve
+ * @param {object} options - Resolution options
+ * @returns {Promise<any>}
+ */
+async function getValueFromFile(ctx, variableString, options) {
+  const opts = options || {}
+  const syntax = opts.asRawText ? ctx.textRefSyntax : ctx.fileRefSyntax
+  // console.log('From file', `"${variableString}"`)
+  let matchedFileString = variableString.match(syntax)[0]
+  // console.log('matchedFileString', matchedFileString)
+
+  // Get function input params if any supplied https://regex101.com/r/qlNFVm/1
+  // var funcParamsRegex = /(\w+)\s*\(((?:[^()]+)*)?\s*\)\s*/g
+  var funcParamsRegex = /(\w+)\s*\(((?:[^()]+)*)?\s*\)/g
+  // tighter (?<![.\w-])\b(\w+)\s*\(((?:[^()]+)*)?\s*\)\s*
+  var hasParams = funcParamsRegex.exec(matchedFileString)
+
+  let argsToPass = []
+  if (hasParams) {
+    const splitter = splitCsv(hasParams[2])
+    const argsFound = splitter.map((arg) => {
+      const cleanArg = trim(arg).replace(/^'|"/, '').replace(/'|"$/, '')
+      return cleanArg
+    })
+    // console.log('argsFound', argsFound)
+
+    // If function has more arguments than file path
+    if (argsFound.length && argsFound.length > 1) {
+      matchedFileString = argsFound[0]
+      argsToPass = argsFound.filter((arg, i) => {
+        return i !== 0
+      })
+    }
+  }
+  // console.log('argsToPass', argsToPass)
+
+  const fileDetails = resolveFilePathFromMatch(matchedFileString, syntax, ctx.configPath)
+  // console.log('fileDetails', fileDetails)
+
+  const { fullFilePath, resolvedPath, relativePath } = fileDetails
+
+  const exists = fs.existsSync(fullFilePath)
+
+  ctx.fileRefsFound.push({
+    // location: options.context.path.join('.'),
+    filePath: fullFilePath,
+    relativePath,
+    resolvedVariableString: options.context.value,
+    originalVariableString: options.context.originalSource,
+    containsVariables: options.context.value !== options.context.originalSource,
+    exists,
+  })
+
+  let fileExtension = resolvedPath.split('.')
+
+  fileExtension = fileExtension[fileExtension.length - 1].toLowerCase()
+
+  // Validate file exists
+  if (!exists) {
+    const originalVar = options.context && options.context.originalSource
+
+    const findNestedResult = findNestedVariables(
+      originalVar,
+      ctx.variableSyntax,
+      ctx.variablesKnownTypes,
+      options.context.path,
+      ctx.variableTypes
+    )
+    // console.log('findNestedResult', findNestedResult)
+    let hasFallback = false
+    if (findNestedResult) {
+      const varDetails = findNestedResult[0]
+      // console.log('varDetails', varDetails)
+      hasFallback = varDetails.hasFallback
+    }
+
+    // check if original var has fallback value
+    // console.log('NO FILE FOUND', fullFilePath)
+    // console.log('variableString', variableString)
+
+    if (!hasFallback && !ctx.opts.allowUnknownFileRefs) {
+      const errorMsg = makeBox({
+        title: `File Not Found in ${originalVar}`,
+        minWidth: '100%',
+        text: `Variable ${variableString} cannot resolve due to missing file.
+
+File not found ${fullFilePath}
+
+Default fallback value will be used if provided.
+
+${JSON.stringify(options.context, null, 2)}`,
+  })
+      console.log(errorMsg)
+    }
+    // TODO maybe reject. YAML does not allow for null/undefined values
+    // return Promise.reject(new Error(errorMsg))
+    return Promise.resolve(undefined)
+  }
+
+  let valueToPopulate
+
+  const variableFileContents = fs.readFileSync(fullFilePath, 'utf-8')
+
+  /* handle case for referencing raw JS files to inline them */
+  if (argsToPass.length
+    && (argsToPass && argsToPass[0] && argsToPass[0].toLowerCase() === 'raw')
+    || opts.asRawText
+  ) {
+    // Encode foo() to foo__PH_PAREN_OPEN__) to avoid function collisions
+    valueToPopulate = encodeJsSyntax(variableFileContents)
+    return Promise.resolve(valueToPopulate)
+  }
+
+  // Build context for executable files
+  const valueForFunction = {
+    originalConfig: ctx.originalConfig,
+    config: ctx.config,
+    opts: ctx.opts,
+  }
+
+  // Process JS files
+  if (fileExtension === 'js' || fileExtension === 'cjs') {
+    const jsFile = require(fullFilePath)
+    const { moduleName } = parseModuleReference(variableString, matchedFileString)
+    const returnValueFunction = moduleName ? jsFile[moduleName] : jsFile
+
+    return processExecutableFile({
+      fileModule: jsFile,
+      returnValueFunction,
+      valueForFunction,
+      argsToPass,
+      variableString,
+      matchedFileString,
+      relativePath,
+      fileType: 'javascript',
+      getDeeperValue: ctx.getDeeperValue
+    })
+  }
+
+  if (fileExtension === 'ts' || fileExtension === 'tsx' || fileExtension === 'mts' || fileExtension === 'cts') {
+    const { executeTypeScriptFile } = require('../parsers/typescript')
+    const { moduleName } = parseModuleReference(variableString, matchedFileString)
+
+    try {
+      const tsFile = await executeTypeScriptFile(fullFilePath, { dynamicArgs: () => argsToPass })
+      let returnValueFunction = tsFile.config || tsFile.default || tsFile
+      if (moduleName) {
+        returnValueFunction = tsFile[moduleName]
+      }
+
+      return processExecutableFile({
+        fileModule: tsFile,
+        returnValueFunction,
+        valueForFunction,
+        argsToPass,
+        variableString,
+        matchedFileString,
+        relativePath,
+        fileType: 'TypeScript',
+        getDeeperValue: ctx.getDeeperValue
+      })
+    } catch (err) {
+      return Promise.reject(new Error(`Error processing TypeScript file: ${err.message}`))
+    }
+  }
+
+  if (fileExtension === 'mjs' || fileExtension === 'esm') {
+    const { executeESMFile } = require('../parsers/esm')
+    const { moduleName } = parseModuleReference(variableString, matchedFileString)
+
+    try {
+      const esmFile = await executeESMFile(fullFilePath, { dynamicArgs: () => argsToPass })
+      let returnValueFunction = esmFile.config || esmFile.default || esmFile
+      if (moduleName) {
+        returnValueFunction = esmFile[moduleName]
+      }
+
+      return processExecutableFile({
+        fileModule: esmFile,
+        returnValueFunction,
+        valueForFunction,
+        argsToPass,
+        variableString,
+        matchedFileString,
+        relativePath,
+        fileType: 'ESM',
+        getDeeperValue: ctx.getDeeperValue
+      })
+    } catch (err) {
+      return Promise.reject(new Error(`Error processing ESM file: ${err.message}`))
+    }
+  }
+
+  // Process everything except JS, TS, and ESM
+  if (fileExtension !== 'js' && fileExtension !== 'ts' && fileExtension !== 'mjs' && fileExtension !== 'esm') {
+    /* Read initial file */
+    valueToPopulate = variableFileContents
+
+    // File reference has :subKey lookup. Must dig deeper
+    if (matchedFileString !== variableString) {
+      if (fileExtension === 'yml' || fileExtension === 'yaml') {
+        valueToPopulate = JSON.stringify(YAML.parse(valueToPopulate))
+      }
+      if (fileExtension === 'toml' || fileExtension === 'tml') {
+        valueToPopulate = JSON.stringify(TOML.parse(valueToPopulate))
+      }
+      if (fileExtension === 'ini') {
+        valueToPopulate = INI.toJson(valueToPopulate)
+      }
+      // console.log('deep', variableString)
+      // console.log('matchedFileString', matchedFileString)
+      let deepProperties = variableString.replace(matchedFileString, '')
+      // Support both : and . as the separator for sub properties
+      const firstChar = deepProperties.substring(0, 1)
+      if (firstChar !== ':' && firstChar !== '.') {
+        const errorMessage = `Invalid variable syntax when referencing file "${relativePath}" sub properties
+Please use ":" or "." to reference sub properties. ${deepProperties}`
+        return Promise.reject(new Error(errorMessage))
+      }
+      deepProperties = deepProperties.slice(1).split('.')
+      return ctx.getDeeperValue(deepProperties, valueToPopulate)
+    }
+
+    if (fileExtension === 'yml' || fileExtension === 'yaml') {
+      valueToPopulate = YAML.parse(valueToPopulate)
+      return Promise.resolve(valueToPopulate)
+    }
+
+    if (fileExtension === 'toml' || fileExtension === 'tml') {
+      valueToPopulate = TOML.parse(valueToPopulate)
+      return Promise.resolve(valueToPopulate)
+    }
+
+    if (fileExtension === 'ini') {
+      valueToPopulate = INI.parse(valueToPopulate)
+      return Promise.resolve(valueToPopulate)
+    }
+
+    if (fileExtension === 'json' || fileExtension === 'json5') {
+      valueToPopulate = JSON5.parse(valueToPopulate)
+      return Promise.resolve(valueToPopulate)
+    }
+  }
+  // console.log('fall thru', valueToPopulate)
+  return Promise.resolve(valueToPopulate)
+}
+
+/**
+ * Parses variable string to extract module reference path
+ * Supports both : and . as separators for module references
+ * @param {string} variableString - The full variable string
+ * @param {string} matchedFileString - The matched file path portion
+ * @returns {{ variableArray: string[], moduleName: string|null }}
+ */
+function parseModuleReference(variableString, matchedFileString) {
+  let variableArray = variableString.split(':')
+  if (variableArray.length === 1) {
+    const dotIndex = variableString.indexOf(matchedFileString) + matchedFileString.length
+    const afterMatch = variableString.substring(dotIndex)
+    if (afterMatch.startsWith('.')) {
+      variableArray = [variableString.substring(0, dotIndex), afterMatch.substring(1)]
+    }
+  }
+
+  let moduleName = null
+  if (variableArray[1]) {
+    moduleName = variableArray[1].split('.')[0]
+  }
+
+  return { variableArray, moduleName }
+}
+
+/**
+ * Extracts deep properties from variable string after file match
+ * @param {string} variableString - The full variable string
+ * @param {string} matchedFileString - The matched file path portion
+ * @returns {string[]} Array of property keys to traverse
+ */
+function extractDeepProperties(variableString, matchedFileString) {
+  let deepProperties = variableString.replace(matchedFileString, '')
+  deepProperties = deepProperties.slice(1).split('.')
+  deepProperties.splice(0, 1)
+  return deepProperties.map((prop) => trim(prop))
+}
+
+/**
+ * Processes executable file (JS/TS/ESM) and resolves deep properties
+ * @param {object} params - Parameters
+ * @param {object} params.fileModule - The loaded module
+ * @param {Function} params.returnValueFunction - The function to call
+ * @param {object} params.valueForFunction - Context passed to the function
+ * @param {string[]} params.argsToPass - Additional args for the function
+ * @param {string} params.variableString - Original variable string
+ * @param {string} params.matchedFileString - Matched file path
+ * @param {string} params.relativePath - Relative file path for errors
+ * @param {string} params.fileType - Type of file (javascript/TypeScript/ESM)
+ * @param {Function} params.getDeeperValue - Function to resolve nested values
+ * @returns {Promise<any>}
+ */
+async function processExecutableFile({
+  fileModule,
+  returnValueFunction,
+  valueForFunction,
+  argsToPass,
+  variableString,
+  matchedFileString,
+  relativePath,
+  fileType,
+  getDeeperValue
+}) {
+  if (typeof returnValueFunction !== 'function') {
+    const errorMessage = `Invalid variable syntax when referencing file "${relativePath}".
+Check if your ${fileType} is exporting a function that returns a value.`
+    return Promise.reject(new Error(errorMessage))
+  }
+
+  const valueToPopulate = returnValueFunction.call(fileModule, valueForFunction, ...argsToPass)
+
+  return Promise.resolve(valueToPopulate).then((valueToPopulateResolved) => {
+    const deepProperties = extractDeepProperties(variableString, matchedFileString)
+    return getDeeperValue(deepProperties, valueToPopulateResolved).then((deepValueToPopulateResolved) => {
+      if (typeof deepValueToPopulateResolved === 'undefined') {
+        const errorMessage = `Invalid variable syntax when referencing file "${relativePath}".
+Check if your ${fileType} is returning the correct data.`
+        return Promise.reject(new Error(errorMessage))
+      }
+      return Promise.resolve(deepValueToPopulateResolved)
+    })
+  })
+}
+
+module.exports = {
+  getValueFromFile,
+  parseFileContents,
+  parseModuleReference,
+  extractDeepProperties
+}

package/src/resolvers/valueFromGit.js

@@ -5,8 +5,8 @@ const path = require('path')
 const childProcess = require('child_process')
 const GitUrlParse = require('git-url-parse')
 const { functionRegex } = require('../utils/regex')
-const formatFunctionArgs = require('../utils/formatFunctionArgs')
-const { findProjectRoot } = require('../utils/find-project-root')
+const formatFunctionArgs = require('../utils/strings/formatFunctionArgs')
+const { findProjectRoot } = require('../utils/paths/findProjectRoot')
 const GIT_PREFIX = 'git'
 const gitVariableSyntax = RegExp(/^git:/g)
 
@@ -330,6 +330,7 @@ async function getGitRemote(name = 'origin') {
 module.exports = function createGitResolver(cwd) {
   return {
     type: 'git',
+    source: 'readonly',
     prefix: 'git',
     syntax: '${git:valueType}',
     description: `Resolves Git variables. Available valueTypes: ${Object.values(GIT_KEYS).join(', ')}`,

package/src/resolvers/valueFromOptions.js

@@ -14,6 +14,7 @@ function getValueFromOptions(variableString, options) {
 
 module.exports = {
   type: 'options',
+  source: 'user',
   prefix: 'opt',
   syntax: '${opt:flagName}',
   description: 'Resolves CLI option flags. Examples: ${opt:stage}, ${opt:other, "fallbackValue"}',

package/src/resolvers/valueFromString.js

@@ -1,8 +1,9 @@
+const { trimSurroundingQuotes } = require('../utils/strings/quoteUtils')
 
 const stringRefSyntax = RegExp(/(?:('|").*?\1)/g)
 
 function getValueFromString(variableString) {
-  const valueToPopulate = variableString.replace(/^['"]|['"]$/g, '')
+  const valueToPopulate = trimSurroundingQuotes(variableString, false)
   return Promise.resolve(valueToPopulate)
 }
 

package/src/sync.js

@@ -1,8 +1,8 @@
 const path = require('path')
 const fs = require('fs')
 const Configorama = require('./main')
-const getFullPath = require('./utils/getFullFilePath')
-const enrichMetadata = require('./utils/enrichMetadata')
+const getFullPath = require('./utils/paths/getFullFilePath')
+const enrichMetadata = require('./utils/parsing/enrichMetadata')
 
 /**
  * Force synchronous invocation of async API
@@ -54,19 +54,26 @@ module.exports = function configoramaSync(variableSources = []) {
       const metadata = instance.collectVariableMetadata()
 
       // Enrich metadata with resolution tracking data collected during execution
-      const enrichedMetadata = enrichMetadata(
+      const enrichedMetadata = await enrichMetadata(
         metadata,
         instance.resolutionTracking,
         instance.variableSyntax,
         instance.fileRefsFound,
         instance.originalConfig,
         instance.configFilePath,
-        Object.keys(instance.filters)
+        Object.keys(instance.filters),
+        result, // pass resolved config for post-resolution enrichment
+        options,
+        instance.variableTypes
       )
 
       return {
+        variableSyntax: instance.variableSyntax,
+        variableTypes: instance.variableTypes,
         config: result,
-        metadata: enrichedMetadata
+        originalConfig: instance.originalConfig,
+        metadata: enrichedMetadata,
+        resolutionHistory: enrichedMetadata.resolutionHistory,
       }
     }
 

package/src/utils/parsing/arrayToJsonPath.test.js

@@ -0,0 +1,56 @@
+const { test } = require('uvu')
+const assert = require('uvu/assert')
+const { arrayToJsonPath } = require('./arrayToJsonPath')
+
+test('arrayToJsonPath - should handle single string element', () => {
+  const result = arrayToJsonPath(['root'])
+  assert.equal(result, 'root')
+})
+
+test('arrayToJsonPath - should join string elements with dots', () => {
+  const result = arrayToJsonPath(['root', 'child', 'grandchild'])
+  assert.equal(result, 'root.child.grandchild')
+})
+
+test('arrayToJsonPath - should handle numeric indices with brackets', () => {
+  const result = arrayToJsonPath(['array', 0, 'property'])
+  assert.equal(result, 'array[0].property')
+})
+
+test('arrayToJsonPath - should handle mixed string and number paths', () => {
+  const result = arrayToJsonPath(['users', 5, 'name'])
+  assert.equal(result, 'users[5].name')
+})
+
+test('arrayToJsonPath - should handle consecutive numbers', () => {
+  const result = arrayToJsonPath(['matrix', 0, 1])
+  assert.equal(result, 'matrix[0][1]')
+})
+
+test('arrayToJsonPath - should handle single number', () => {
+  const result = arrayToJsonPath([0])
+  assert.equal(result, '0')
+})
+
+test('arrayToJsonPath - should handle complex nested path', () => {
+  const result = arrayToJsonPath(['data', 'items', 3, 'values', 2, 'name'])
+  assert.equal(result, 'data.items[3].values[2].name')
+})
+
+test('arrayToJsonPath - should handle empty array', () => {
+  const result = arrayToJsonPath([])
+  assert.equal(result, '')
+})
+
+test('arrayToJsonPath - should convert number to string for first element', () => {
+  const result = arrayToJsonPath([123, 'property'])
+  assert.equal(result, '123.property')
+})
+
+test('arrayToJsonPath - should handle property names with special chars', () => {
+  const result = arrayToJsonPath(['root', 'my-prop', 'my_prop'])
+  assert.equal(result, 'root.my-prop.my_prop')
+})
+
+// Run all tests
+test.run()