@dotenvx/dotenvx 1.23.0 → 1.24.0

package/CHANGELOG.md

@@ -2,7 +2,31 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
-## [Unreleased](https://github.com/dotenvx/dotenvx/compare/v1.23.0...main)
+## [Unreleased](https://github.com/dotenvx/dotenvx/compare/v1.24.0...main)
+
+## 1.24.0
+
+### Added
+
+* support progressive append/update ([#445](https://github.com/dotenvx/dotenvx/pull/445))
+
+```ini
+FOO=foo
+FOO=${FOO}bar
+# foobar
+```
+
+* support alternate value expansion ([#445](https://github.com/dotenvx/dotenvx/pull/445))
+
+<img width="1608" alt="image" src="https://github.com/user-attachments/assets/fdd55a0a-9b36-4cb3-b0c6-6b019441aef4">
+
+### Changed
+
+* `dotenvx.parse` now maps to dotenvx's internal parser. (prior it was mapping to [dotenv's](https://github.com/motdotla/dotenv))
+
+### Removed
+
+* removed `dotenvx.configDotenv()`. use `dotenvx.config()` ([#445](https://github.com/dotenvx/dotenvx/pull/445))
 
 ## 1.23.0
 

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.23.0",
+  "version": "1.24.0",
   "name": "@dotenvx/dotenvx",
   "description": "a better dotenv–from the creator of `dotenv`",
   "author": "@motdotla",
@@ -31,8 +31,7 @@
     "test-single": "tap run --coverage-report=none tests/cli/actions/decrypt.test.js",
     "testshell": "bash shellspec",
     "prerelease": "npm test && npm run testshell",
-    "release": "standard-version",
-    "patch": "patch-package"
+    "release": "standard-version"
   },
   "funding": "https://dotenvx.com",
   "dependencies": {
@@ -47,9 +46,8 @@
     "which": "^4.0.0"
   },
   "devDependencies": {
+    "@yao-pkg/pkg": "^5.14.2",
     "capture-console": "^1.0.2",
-    "patch-package": "^8.0.0",
-    "pkg": "^5.8.1",
     "proxyquire": "^2.1.3",
     "sinon": "^14.0.1",
     "standard": "^17.1.0",

package/src/lib/helpers/chomp.js

@@ -0,0 +1,5 @@
+function chomp (value) {
+  return value.replace(/[\r\n]+$/, '')
+}
+
+module.exports = chomp

package/src/lib/helpers/execute.js

@@ -1,8 +1,11 @@
 const execa = require('execa')
+/* c8 ignore start */
+const pkgArgs = process.pkg ? { PKG_EXECPATH: '' } : {}
+/* c8 ignore stop */
 
 const execute = {
   execa (command, args, options) {
-    return execa(command, args, options)
+    return execa(command, args, { ...options, env: { ...options.env, ...pkgArgs } })
   }
 }
 

package/src/lib/helpers/parse.js

@@ -0,0 +1,208 @@
+const chomp = require('./chomp')
+const truncate = require('./truncate')
+const decryptValue = require('./decryptValue')
+const resolveEscapeSequences = require('./resolveEscapeSequences')
+const { execSync } = require('child_process')
+
+class Parse {
+  static LINE = /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/mg
+
+  constructor (src, privateKey = null, processEnv = process.env, overload = false) {
+    this.src = src
+    this.privateKey = privateKey
+    this.processEnv = processEnv
+    this.overload = overload
+
+    this.parsed = {}
+    this.preExisted = {}
+    this.injected = {}
+    this.warnings = []
+
+    // for use with progressive expansion
+    this.runningParsed = {}
+  }
+
+  run () {
+    const lines = this.getLines()
+
+    let match
+    while ((match = Parse.LINE.exec(lines)) !== null) {
+      const key = match[1]
+      const value = match[2]
+      const quote = this.quote(value) // must be raw match
+      this.parsed[key] = this.clean(value, quote) // file value
+
+      if (!this.overload && this.inProcessEnv(key)) {
+        this.parsed[key] = this.processEnv[key] // use process.env pre-existing value
+      }
+
+      // decrypt
+      try {
+        this.parsed[key] = this.decrypt(this.parsed[key])
+      } catch (e) {
+        this.warnings.push(this.warning(e, key))
+      }
+
+      // eval empty, double, or backticks
+      if (quote !== "'" && (!this.inProcessEnv(key) || this.processEnv[key] === this.parsed[key])) {
+        this.parsed[key] = this.eval(this.parsed[key])
+      }
+
+      // expand empty, double, or backticks
+      if (quote !== "'") {
+        this.parsed[key] = resolveEscapeSequences(this.expand(this.parsed[key]))
+      }
+
+      // for use with progressive expansion
+      this.runningParsed[key] = this.parsed[key]
+
+      if (Object.prototype.hasOwnProperty.call(this.processEnv, key) && !this.overload) {
+        this.preExisted[key] = this.processEnv[key] // track preExisted
+      } else {
+        this.injected[key] = this.parsed[key] // track injected
+      }
+    }
+
+    return {
+      parsed: this.parsed,
+      processEnv: this.processEnv,
+      injected: this.injected,
+      warnings: this.warnings,
+      preExisted: this.preExisted
+    }
+  }
+
+  trimmer (value) {
+    // Default undefined or null to empty string
+    return (value || '').trim()
+  }
+
+  quote (value) {
+    const v = this.trimmer(value)
+    const maybeQuote = v[0]
+    let q = ''
+    switch (maybeQuote) {
+      // single
+      case "'":
+        q = "'"
+        break
+      // double
+      case '"':
+        q = '"'
+        break
+      // backtick
+      case '`':
+        q = '`'
+        break
+      // empty
+      default:
+        q = ''
+    }
+
+    return q
+  }
+
+  clean (value, _quote) {
+    let v = this.trimmer(value)
+
+    // Remove surrounding quotes
+    v = v.replace(/^(['"`])([\s\S]*)\1$/mg, '$2')
+
+    // Expand newlines if double quoted
+    if (_quote === '"') {
+      v = v.replace(/\\n/g, '\n')
+      v = v.replace(/\\r/g, '\r')
+    }
+
+    return v
+  }
+
+  decrypt (value) {
+    return decryptValue(value, this.privateKey)
+  }
+
+  eval (value) {
+    const matches = value.match(/\$\([^()]+\)/) || []
+
+    return matches.reduce(function (newValue, match) {
+      const command = match.substring(2, match.length - 1) // get command
+      const value = chomp(execSync(command).toString()) // execute command
+      return newValue.replace(match, value) // replace with command value
+    }, value)
+  }
+
+  expand (value) {
+    let env = { ...this.runningParsed, ...this.processEnv } // typically process.env wins
+    if (this.overload) {
+      env = { ...this.processEnv, ...this.runningParsed } // parsed wins
+    }
+
+    const regex = /(?<!\\)\${([^{}]+)}|(?<!\\)\$([A-Za-z_][A-Za-z0-9_]*)/g
+
+    let result = value
+    let match
+    const seen = new Set() // self-referential checker
+
+    while ((match = regex.exec(result)) !== null) {
+      seen.add(result)
+
+      const [template, bracedExpression, unbracedExpression] = match
+      const expression = bracedExpression || unbracedExpression
+
+      // match the operators `:+`, `+`, `:-`, and `-`
+      const opRegex = /(:\+|\+|:-|-)/
+      // find first match
+      const opMatch = expression.match(opRegex)
+      const splitter = opMatch ? opMatch[0] : null
+
+      const r = expression.split(splitter)
+
+      let key
+      let defaultValue
+      let value
+
+      if ([':+', '+'].includes(splitter)) {
+        key = r.shift()
+        defaultValue = env[key] ? r.join(splitter) : ''
+        value = null
+      } else {
+        key = r.shift()
+        defaultValue = r.join(splitter)
+        value = env[key]
+      }
+
+      if (value) {
+        // self-referential check
+        if (seen.has(value)) {
+          result = result.replace(template, defaultValue)
+        } else {
+          result = result.replace(template, value)
+        }
+      } else {
+        result = result.replace(template, defaultValue)
+      }
+
+      regex.lastIndex = 0 // reset regex search position to re-evaluate after each replacement
+    }
+
+    return result
+  }
+
+  inProcessEnv (key) {
+    return Object.prototype.hasOwnProperty.call(this.processEnv, key)
+  }
+
+  getLines () {
+    return (this.src || '').toString().replace(/\r\n?/mg, '\n') // Convert buffer to string and Convert line breaks to same format
+  }
+
+  warning (e, key) {
+    const warning = new Error(`[${e.code}] could not decrypt ${key} using private key '${truncate(this.privateKey)}'`)
+    warning.code = e.code
+    warning.help = `[${e.code}] ? ${e.message}`
+
+    return warning
+  }
+}
+
+module.exports = Parse

package/src/lib/helpers/resolveEscapeSequences.js

@@ -0,0 +1,5 @@
+function resolveEscapeSequences (value) {
+  return value.replace(/\\\$/g, '$')
+}
+
+module.exports = resolveEscapeSequences

package/src/lib/main.d.ts

@@ -124,17 +124,6 @@ export interface DotenvPopulateInput {
  */
 export function config(options?: DotenvConfigOptions): DotenvConfigOutput;
 
-/**
- * Loads `.env` file contents into process.env.
- *
- * @see https://dotenvx.com/docs
- *
- * @param options - additional options. example: `{ path: './custom/path', encoding: 'latin1', debug: true, override: false }`
- * @returns an object with a `parsed` key if successful or `error` key if an error occurred. example: { parsed: { KEY: 'value' } }
- *
- */
-export function configDotenv(options?: DotenvConfigOptions): DotenvConfigOutput;
-
 /**
  * Decrypt ciphertext
  *

package/src/lib/main.js

@@ -1,6 +1,5 @@
 // @ts-check
 const path = require('path')
-const dotenv = require('dotenv')
 
 // shared
 const { setLogLevel, logger } = require('./../shared/logger')
@@ -16,8 +15,7 @@ const Genexample = require('./services/genexample')
 // helpers
 const conventions = require('./helpers/conventions')
 const dotenvOptionPaths = require('./helpers/dotenvOptionPaths')
-
-// proxies to dotenv
+const Parse = require('./helpers/parse')
 
 /** @type {import('./main').config} */
 const config = function (options = {}) {
@@ -162,14 +160,23 @@ const config = function (options = {}) {
   }
 }
 
-/** @type {import('./main').configDotenv} */
-const configDotenv = function (options) {
-  return dotenv.configDotenv(options)
-}
-
 /** @type {import('./main').parse} */
-const parse = function (src) {
-  return dotenv.parse(src)
+const parse = function (src, options = {}) {
+  // allow user to set processEnv to read from
+  let processEnv = process.env
+  if (options && options.processEnv != null) {
+    processEnv = options.processEnv
+  }
+
+  // private decryption key
+  const privateKey = null // implement later
+
+  // overload
+  const overload = options.overload || options.override
+
+  const { parsed } = new Parse(src, privateKey, processEnv, overload).run()
+
+  return parsed
 }
 
 /** @type {import('./main').ls} */
@@ -201,7 +208,6 @@ const keypair = function (envFile, key) {
 module.exports = {
   // dotenv proxies
   config,
-  configDotenv,
   parse,
   // actions related
   ls,

package/src/lib/services/run.js

@@ -6,9 +6,8 @@ const TYPE_ENV = 'env'
 const TYPE_ENV_FILE = 'envFile'
 const TYPE_ENV_VAULT_FILE = 'envVaultFile'
 
-const inject = require('./../helpers/inject')
 const decrypt = require('./../helpers/decrypt')
-const parseDecryptEvalExpand = require('./../helpers/parseDecryptEvalExpand')
+const Parse = require('./../helpers/parse')
 const parseEnvironmentFromDotenvKey = require('./../helpers/parseEnvironmentFromDotenvKey')
 const detectEncoding = require('./../helpers/detectEncoding')
 const findPrivateKey = require('./../helpers/findPrivateKey')
@@ -60,15 +59,16 @@ class Run {
     row.string = env
 
     try {
-      const { parsed, processEnv, warnings } = parseDecryptEvalExpand(env, null, this.processEnv)
+      const { parsed, warnings, injected, preExisted } = new Parse(env, null, this.processEnv, this.overload).run()
       row.parsed = parsed
       row.warnings = warnings
-      this.readableStrings.add(env)
-
-      const { injected, preExisted } = this._inject(processEnv, parsed, this.overload, this.processEnv)
       row.injected = injected
       row.preExisted = preExisted
 
+      this.inject(row.parsed) // inject
+
+      this.readableStrings.add(env)
+
       for (const key of Object.keys(injected)) {
         this.uniqueInjectedKeys.add(key) // track uniqueInjectedKeys across multiple files
       }
@@ -91,14 +91,15 @@ class Run {
       this.readableFilepaths.add(envFilepath)
 
       const privateKey = findPrivateKey(envFilepath)
-      const { parsed, processEnv, warnings } = parseDecryptEvalExpand(src, privateKey, this.processEnv)
+      const { parsed, warnings, injected, preExisted } = new Parse(src, privateKey, this.processEnv, this.overload).run()
+
       row.parsed = parsed
       row.warnings = warnings
-
-      const { injected, preExisted } = this._inject(processEnv, parsed, this.overload, this.processEnv)
       row.injected = injected
       row.preExisted = preExisted
 
+      this.inject(row.parsed) // inject
+
       for (const key of Object.keys(injected)) {
         this.uniqueInjectedKeys.add(key) // track uniqueInjectedKeys across multiple files
       }
@@ -161,14 +162,14 @@ class Run {
 
     try {
       // parse this. it's the equivalent of the .env file
-      const { parsed, processEnv, warnings } = parseDecryptEvalExpand(decrypted, null, this.processEnv)
+      const { parsed, warnings, injected, preExisted } = new Parse(decrypted, null, this.processEnv, this.overload).run()
       row.parsed = parsed
       row.warnings = warnings
-
-      const { injected, preExisted } = this._inject(processEnv, parsed, this.overload, this.processEnv)
       row.injected = injected
       row.preExisted = preExisted
 
+      this.inject(row.parsed) // inject
+
       for (const key of Object.keys(injected)) {
         this.uniqueInjectedKeys.add(key) // track uniqueInjectedKeys across multiple files
       }
@@ -179,8 +180,10 @@ class Run {
     this.processedEnvs.push(row)
   }
 
-  _inject (clonedProcessEnv, parsed, overload, processEnv) {
-    return inject(clonedProcessEnv, parsed, overload, processEnv)
+  inject (parsed) {
+    for (const key of Object.keys(parsed)) {
+      this.processEnv[key] = parsed[key] // inject to process.env
+    }
   }
 
   // handle scenario for comma separated keys - for use with key rotation

package/src/lib/helpers/dotenvEval.js

@@ -1,54 +0,0 @@
-const { execSync } = require('child_process')
-
-function _chomp (value) {
-  return value.replace(/[\r\n]+$/, '')
-}
-
-function interpolate (value, processEnv, parsed) {
-  const matches = value.match(/\$\([^()]+\)/) || []
-
-  return matches.reduce(function (newValue, match) {
-    // get command
-    const command = match.substring(2, match.length - 1)
-    // execute command
-    const value = _chomp(execSync(command).toString())
-    // replace with command value
-    return newValue.replace(match, value)
-  }, value)
-}
-
-function evaluate (options) {
-  let processEnv = process.env
-  if (options && options.processEnv != null) {
-    processEnv = options.processEnv
-  }
-
-  for (const key in options.parsed) {
-    let value = options.parsed[key]
-
-    const inProcessEnv = Object.prototype.hasOwnProperty.call(processEnv, key)
-
-    if (inProcessEnv) {
-      if (processEnv[key] === options.parsed[key]) {
-        // assume was set to processEnv from the .env file if the values match and therefore interpolate
-        value = interpolate(value, processEnv, options.parsed)
-      } else {
-        // do not interpolate - assume processEnv had the intended value even if containing a $.
-        value = processEnv[key]
-      }
-    } else {
-      // not inProcessEnv so assume interpolation for this .env key
-      value = interpolate(value, processEnv, options.parsed)
-    }
-
-    options.parsed[key] = value
-  }
-
-  for (const processKey in options.parsed) {
-    processEnv[processKey] = options.parsed[processKey]
-  }
-
-  return options
-}
-
-module.exports.eval = evaluate

package/src/lib/helpers/dotenvExpand.js

@@ -1,73 +0,0 @@
-function _resolveEscapeSequences (value) {
-  return value.replace(/\\\$/g, '$')
-}
-
-function interpolate (value, env) {
-  const regex = /(?<!\\)\${([^{}]+)}|(?<!\\)\$([A-Za-z_][A-Za-z0-9_]*)/g
-
-  let result = value
-  let match
-  const seen = new Set() // self-referential checker
-
-  while ((match = regex.exec(result)) !== null) {
-    seen.add(result)
-
-    const [template, bracedExpression, unbracedExpression] = match
-    const expression = bracedExpression || unbracedExpression
-    const r = expression.split(/:-|-/)
-    const key = r.shift()
-    const defaultValue = r.join('-')
-    const value = env[key]
-
-    if (value) {
-      // self-referential check
-      if (seen.has(value)) {
-        result = result.replace(template, defaultValue)
-      } else {
-        result = result.replace(template, value)
-      }
-    } else {
-      result = result.replace(template, defaultValue)
-    }
-
-    regex.lastIndex = 0 // reset regex search position to re-evaluate after each replacement
-  }
-
-  return result
-}
-
-function expand (options) {
-  const processEnv = options.processEnv || {}
-  const parsed = options.parsed || {}
-
-  const combined = { ...processEnv, ...parsed }
-  const combinedReversed = { ...parsed, ...processEnv }
-
-  for (const key in parsed) {
-    const value = parsed[key]
-
-    // interpolate using both file and processEnv (file interpolation wins. used for --overload later)
-    const fileValue = _resolveEscapeSequences(interpolate(value, combined))
-    parsed[key] = fileValue
-
-    if (fileValue === _resolveEscapeSequences(value)) {
-      continue // no change means no expansion, move on
-    }
-
-    if (processEnv[key]) {
-      continue // already has a value in processEnv, move on
-    }
-
-    const processEnvValue = interpolate(value, combinedReversed) // could be empty string ''
-    if (processEnvValue) {
-      processEnv[key] = _resolveEscapeSequences(processEnvValue) // set it
-    }
-  }
-
-  return {
-    parsed,
-    processEnv
-  }
-}
-
-module.exports.expand = expand

package/src/lib/helpers/inject.js

@@ -1,27 +0,0 @@
-function inject (clonedProcessEnv = {}, parsed = {}, overload = false, processEnv = process.env) {
-  const injected = {}
-  const preExisted = {}
-
-  // set processEnv
-  for (const key of Object.keys(parsed)) {
-    if (Object.prototype.hasOwnProperty.call(clonedProcessEnv, key)) {
-      if (overload === true) {
-        processEnv[key] = parsed[key]
-        injected[key] = parsed[key] // track injected key/value
-      } else {
-        processEnv[key] = clonedProcessEnv[key]
-        preExisted[key] = clonedProcessEnv[key] // track preExisted key/value
-      }
-    } else {
-      processEnv[key] = parsed[key]
-      injected[key] = parsed[key] // track injected key/value
-    }
-  }
-
-  return {
-    injected,
-    preExisted
-  }
-}
-
-module.exports = inject

package/src/lib/helpers/parseDecryptEvalExpand.js

@@ -1,84 +0,0 @@
-const dotenv = require('dotenv')
-const dotenvEval = require('./dotenvEval')
-const dotenvExpand = require('./dotenvExpand')
-const decryptValue = require('./decryptValue')
-const truncate = require('./truncate')
-const quotes = require('./quotes')
-
-function warning (e, key, privateKey = null) {
-  const warning = new Error(`[${e.code}] could not decrypt ${key} using private key '${truncate(privateKey)}'`)
-  warning.code = e.code
-  warning.help = `[${e.code}] ? ${e.message}`
-
-  return warning
-}
-
-function parseDecryptEvalExpand (src, privateKey = null, processEnv = process.env) {
-  const warnings = []
-
-  // parse and quotes
-  const parsed = dotenv.parse(src)
-  const _quotes = quotes(src)
-  const originalParsed = { ...parsed }
-  const originalProcessEnv = { ...processEnv }
-  for (const key in parsed) {
-    try {
-      const decryptedValue = decryptValue(parsed[key], privateKey)
-      parsed[key] = decryptedValue
-    } catch (_e) {
-      // do nothing. warnings tracked further below.
-    }
-  }
-
-  // eval parsed only. do NOT eval process.env ever. too risky/dangerous.
-  const inputParsed = {
-    processEnv: {},
-    parsed
-  }
-  const evaled = dotenvEval.eval(inputParsed).parsed
-
-  // expanded
-  const inputEvaled = {
-    processEnv,
-    parsed: evaled
-  }
-  const expanded = dotenvExpand.expand(inputEvaled)
-
-  for (const key in expanded.parsed) {
-    // unset eval and expansion for single quotes
-    if (_quotes[key] === "'") {
-      expanded.parsed[key] = originalParsed[key] // reset to original
-    }
-
-    try {
-      const decryptedValue = decryptValue(expanded.parsed[key], privateKey)
-      expanded.parsed[key] = decryptedValue
-    } catch (e) {
-      warnings.push(warning(e, key, privateKey))
-    }
-  }
-
-  for (const key in processEnv) {
-    // unset eval and expansion for single quotes
-    if (_quotes[key] === "'") {
-      processEnv[key] = originalProcessEnv[key] || originalParsed[key] // reset to original
-    }
-
-    try {
-      const decryptedValue = decryptValue(processEnv[key], privateKey)
-      processEnv[key] = decryptedValue
-    } catch (e) {
-      warnings.push(warning(e, key, privateKey))
-    }
-  }
-
-  // for logging only log the original keys existing in parsed. this feels unnecessarily complex - like dotenv-expand should support the ability to inject additional `process.env` or objects as it sees fit to the object it wants to expand
-  const result = {}
-  for (const key in parsed) {
-    result[key] = expanded.parsed[key]
-  }
-
-  return { parsed: result, processEnv, warnings }
-}
-
-module.exports = parseDecryptEvalExpand