dotenv 2.8.1 → 3.2.0

data/lib/dotenv/parser.rb

@@ -2,77 +2,96 @@ require "dotenv/substitutions/variable"
 require "dotenv/substitutions/command" if RUBY_VERSION > "1.8.7"
 
 module Dotenv
+  # Error raised when encountering a syntax error while parsing a .env file.
   class FormatError < SyntaxError; end
 
-  # This class enables parsing of a string for key value pairs to be returned
-  # and stored in the Environment. It allows for variable substitutions and
-  # exporting of variables.
+  # Parses the `.env` file format into key/value pairs.
+  # It allows for variable substitutions, command substitutions, and exporting of variables.
   class Parser
-    @substitutions =
-      [Dotenv::Substitutions::Variable, Dotenv::Substitutions::Command]
+    @substitutions = [
+      Dotenv::Substitutions::Command,
+      Dotenv::Substitutions::Variable
+    ]
 
     LINE = /
-      (?:^|\A)              # beginning of line
-      \s*                   # leading whitespace
-      (?:export\s+)?        # optional export
-      ([\w.]+)              # key
-      (?:\s*=\s*?|:\s+?)    # separator
-      (                     # optional value begin
-        \s*'(?:\\'|[^'])*'  #   single quoted value
-        |                   #   or
-        \s*"(?:\\"|[^"])*"  #   double quoted value
-        |                   #   or
-        [^\#\r\n]+          #   unquoted value
-      )?                    # value end
-      \s*                   # trailing whitespace
-      (?:\#.*)?             # optional comment
-      (?:$|\z)              # end of line
+      (?:^|\A)                # beginning of line
+      \s*                     # leading whitespace
+      (?<export>export\s+)?   # optional export
+      (?<key>[\w.]+)          # key
+      (?:                     # optional separator and value
+        (?:\s*=\s*?|:\s+?)    #   separator
+        (?<value>             #   optional value begin
+          \s*'(?:\\'|[^'])*'  #     single quoted value
+          |                   #     or
+          \s*"(?:\\"|[^"])*"  #     double quoted value
+          |                   #     or
+          [^\#\n]+            #     unquoted value
+        )?                    #   value end
+      )?                      # separator and value end
+      \s*                     # trailing whitespace
+      (?:\#.*)?               # optional comment
+      (?:$|\z)                # end of line
     /x
 
+    QUOTED_STRING = /\A(['"])(.*)\1\z/m
+
     class << self
       attr_reader :substitutions
 
-      def call(string, is_load = false)
-        new(string, is_load).call
+      def call(...)
+        new(...).call
       end
     end
 
-    def initialize(string, is_load = false)
-      @string = string
+    def initialize(string, overwrite: false)
+      # Convert line breaks to same format
+      @string = string.gsub(/\r\n?/, "\n")
       @hash = {}
-      @is_load = is_load
+      @overwrite = overwrite
     end
 
     def call
-      # Convert line breaks to same format
-      lines = @string.gsub(/\r\n?/, "\n")
-      # Process matches
-      lines.scan(LINE).each do |key, value|
-        @hash[key] = parse_value(value || "")
-      end
-      # Process non-matches
-      lines.gsub(LINE, "").split(/[\n\r]+/).each do |line|
-        parse_line(line)
+      @string.scan(LINE) do
+        match = $LAST_MATCH_INFO
+
+        if existing?(match[:key])
+          # Use value from already defined variable
+          @hash[match[:key]] = ENV[match[:key]]
+        elsif match[:export] && !match[:value]
+          # Check for exported variable with no value
+          if !@hash.member?(match[:key])
+            raise FormatError, "Line #{match.to_s.inspect} has an unset variable"
+          end
+        else
+          @hash[match[:key]] = parse_value(match[:value] || "")
+        end
       end
+
       @hash
     end
 
     private
 
-    def parse_line(line)
-      if line.split.first == "export"
-        if variable_not_set?(line)
-          raise FormatError, "Line #{line.inspect} has an unset variable"
-        end
-      end
+    # Determine if a variable is already defined and should not be overwritten.
+    def existing?(key)
+      !@overwrite && key != "DOTENV_LINEBREAK_MODE" && ENV.key?(key)
     end
 
     def parse_value(value)
       # Remove surrounding quotes
-      value = value.strip.sub(/\A(['"])(.*)\1\z/m, '\2')
+      value = value.strip.sub(QUOTED_STRING, '\2')
       maybe_quote = Regexp.last_match(1)
-      value = unescape_value(value, maybe_quote)
-      perform_substitutions(value, maybe_quote)
+
+      # Expand new lines in double quoted values
+      value = expand_newlines(value) if maybe_quote == '"'
+
+      # Unescape characters and performs substitutions unless value is single quoted
+      if maybe_quote != "'"
+        value = unescape_characters(value)
+        self.class.substitutions.each { |proc| value = proc.call(value, @hash) }
+      end
+
+      value
     end
 
     def unescape_characters(value)
@@ -80,30 +99,11 @@ module Dotenv
     end
 
     def expand_newlines(value)
-      value.gsub('\n', "\n").gsub('\r', "\r")
-    end
-
-    def variable_not_set?(line)
-      !line.split[1..-1].all? { |var| @hash.member?(var) }
-    end
-
-    def unescape_value(value, maybe_quote)
-      if maybe_quote == '"'
-        unescape_characters(expand_newlines(value))
-      elsif maybe_quote.nil?
-        unescape_characters(value)
+      if (@hash["DOTENV_LINEBREAK_MODE"] || ENV["DOTENV_LINEBREAK_MODE"]) == "legacy"
+        value.gsub('\n', "\n").gsub('\r', "\r")
       else
-        value
+        value.gsub('\n', "\\\\\\n").gsub('\r', "\\\\\\r")
       end
     end
-
-    def perform_substitutions(value, maybe_quote)
-      if maybe_quote != "'"
-        self.class.substitutions.each do |proc|
-          value = proc.call(value, @hash, @is_load)
-        end
-      end
-      value
-    end
   end
 end

data/lib/dotenv/rails-now.rb

@@ -0,0 +1,10 @@
+# If you use gems that require environment variables to be set before they are
+# loaded, then list `dotenv` in the `Gemfile` before those other gems and
+# require `dotenv/load`.
+#
+#     gem "dotenv", require: "dotenv/load"
+#     gem "gem-that-requires-env-variables"
+#
+
+require "dotenv/load"
+warn '[DEPRECATION] `require "dotenv/rails-now"` is deprecated. Use `require "dotenv/load"` instead.', caller(1..1).first

data/lib/dotenv/rails.rb

@@ -0,0 +1,111 @@
+# Since rubygems doesn't support optional dependencies, we have to manually check
+unless Gem::Requirement.new(">= 6.1").satisfied_by?(Gem::Version.new(Rails.version))
+  warn "dotenv 3.0 only supports Rails 6.1 or later. Use dotenv ~> 2.0."
+  return
+end
+
+require "dotenv/replay_logger"
+require "dotenv/log_subscriber"
+
+Dotenv.instrumenter = ActiveSupport::Notifications
+
+# Watch all loaded env files with Spring
+ActiveSupport::Notifications.subscribe("load.dotenv") do |*args|
+  if defined?(Spring) && Spring.respond_to?(:watch)
+    event = ActiveSupport::Notifications::Event.new(*args)
+    Spring.watch event.payload[:env].filename if Rails.application
+  end
+end
+
+module Dotenv
+  # Rails integration for using Dotenv to load ENV variables from a file
+  class Rails < ::Rails::Railtie
+    delegate :files, :files=, :overwrite, :overwrite=, :autorestore, :autorestore=, :logger, to: "config.dotenv"
+
+    def initialize
+      super
+      config.dotenv = ActiveSupport::OrderedOptions.new.update(
+        # Rails.logger is not available yet, so we'll save log messages and replay them when it is
+        logger: Dotenv::ReplayLogger.new,
+        overwrite: false,
+        files: [
+          ".env.#{env}.local",
+          (".env.local" unless env.test?),
+          ".env.#{env}",
+          ".env"
+        ].compact,
+        autorestore: env.test? && !defined?(ClimateControl) && !defined?(IceAge)
+      )
+    end
+
+    # Public: Load dotenv
+    #
+    # This will get called during the `before_configuration` callback, but you
+    # can manually call `Dotenv::Rails.load` if you needed it sooner.
+    def load
+      Dotenv.load(*files.map { |file| root.join(file).to_s }, overwrite: overwrite)
+    end
+
+    def overload
+      deprecator.warn("Dotenv::Rails.overload is deprecated. Set `Dotenv::Rails.overwrite = true` and call Dotenv::Rails.load instead.")
+      Dotenv.load(*files.map { |file| root.join(file).to_s }, overwrite: true)
+    end
+
+    # Internal: `Rails.root` is nil in Rails 4.1 before the application is
+    # initialized, so this falls back to the `RAILS_ROOT` environment variable,
+    # or the current working directory.
+    def root
+      ::Rails.root || Pathname.new(ENV["RAILS_ROOT"] || Dir.pwd)
+    end
+
+    # Set a new logger and replay logs
+    def logger=(new_logger)
+      logger.replay new_logger if logger.is_a?(ReplayLogger)
+      config.dotenv.logger = new_logger
+    end
+
+    # The current environment that the app is running in.
+    #
+    # When running `rake`, the Rails application is initialized in development, so we have to
+    # check which rake tasks are being run to determine the environment.
+    #
+    # See https://github.com/bkeepers/dotenv/issues/219
+    def env
+      @env ||= if defined?(Rake.application) && Rake.application.top_level_tasks.grep(TEST_RAKE_TASKS).any?
+        env = Rake.application.options.show_tasks ? "development" : "test"
+        ActiveSupport::EnvironmentInquirer.new(env)
+      else
+        ::Rails.env
+      end
+    end
+    TEST_RAKE_TASKS = /^(default$|test(:|$)|parallel:spec|spec(:|$))/
+
+    def deprecator # :nodoc:
+      @deprecator ||= ActiveSupport::Deprecation.new
+    end
+
+    # Rails uses `#method_missing` to delegate all class methods to the
+    # instance, which means `Kernel#load` gets called here. We don't want that.
+    def self.load
+      instance.load
+    end
+
+    initializer "dotenv", after: :initialize_logger do |app|
+      if logger.is_a?(ReplayLogger)
+        self.logger = ActiveSupport::TaggedLogging.new(::Rails.logger).tagged("dotenv")
+      end
+    end
+
+    initializer "dotenv.deprecator" do |app|
+      app.deprecators[:dotenv] = deprecator if app.respond_to?(:deprecators)
+    end
+
+    initializer "dotenv.autorestore" do |app|
+      require "dotenv/autorestore" if autorestore
+    end
+
+    config.before_configuration { load }
+  end
+
+  Railtie = ActiveSupport::Deprecation::DeprecatedConstantProxy.new("Dotenv::Railtie", "Dotenv::Rails", Dotenv::Rails.deprecator)
+end

data/lib/dotenv/replay_logger.rb

@@ -0,0 +1,20 @@
+module Dotenv
+  # A logger that can be used before the apps real logger is initialized.
+  class ReplayLogger < Logger
+    def initialize
+      super(nil) # Doesn't matter what this is, it won't be used.
+      @logs = []
+    end
+
+    # Override the add method to store logs so we can replay them to a real logger later.
+    def add(*args, &block)
+      @logs.push([args, block])
+    end
+
+    # Replay the store logs to a real logger.
+    def replay(logger)
+      @logs.each { |args, block| logger.add(*args, &block) }
+      @logs.clear
+    end
+  end
+end

data/lib/dotenv/substitutions/command.rb

@@ -20,7 +20,7 @@ module Dotenv
           )
         /x
 
-        def call(value, _env, _is_load)
+        def call(value, env)
           # Process interpolated shell commands
           value.gsub(INTERPOLATED_SHELL_COMMAND) do |*|
             # Eliminate opening and closing parentheses
@@ -28,10 +28,10 @@ module Dotenv
 
             if $LAST_MATCH_INFO[:backslash]
               # Command is escaped, don't replace it.
-              $LAST_MATCH_INFO[0][1..-1]
+              $LAST_MATCH_INFO[0][1..]
             else
               # Execute the command and return the value
-              `#{command}`.chomp
+              `#{Variable.call(command, env)}`.chomp
             end
           end
         end

data/lib/dotenv/substitutions/variable.rb

@@ -12,29 +12,23 @@ module Dotenv
         VARIABLE = /
           (\\)?         # is it escaped with a backslash?
           (\$)          # literal $
-          (?!\()        # shouldnt be followed by paranthesis
+          (?!\()        # shouldn't be followed by parenthesis
           \{?           # allow brace wrapping
           ([A-Z0-9_]+)? # optional alpha nums
           \}?           # closing brace
         /xi
 
-        def call(value, env, is_load)
-          combined_env = is_load ? env.merge(ENV) : ENV.to_h.merge(env)
+        def call(value, env)
           value.gsub(VARIABLE) do |variable|
             match = $LAST_MATCH_INFO
-            substitute(match, variable, combined_env)
-          end
-        end
-
-        private
 
-        def substitute(match, variable, env)
-          if match[1] == "\\"
-            variable[1..-1]
-          elsif match[3]
-            env.fetch(match[3], "")
-          else
-            variable
+            if match[1] == "\\"
+              variable[1..]
+            elsif match[3]
+              env[match[3]] || ENV[match[3]] || ""
+            else
+              variable
+            end
           end
         end
       end

data/lib/dotenv/template.rb

@@ -10,17 +10,35 @@ module Dotenv
       File.open(@env_file, "r") do |env_file|
         File.open("#{@env_file}.template", "w") do |env_template|
           env_file.each do |line|
-            env_template.puts template_line(line)
+            if is_comment?(line)
+              env_template.puts line
+            elsif (var = var_defined?(line))
+              if line.match(EXPORT_COMMAND)
+                env_template.puts "export #{var}=#{var}"
+              else
+                env_template.puts "#{var}=#{var}"
+              end
+            elsif line_blank?(line)
+              env_template.puts
+            end
           end
         end
       end
     end
 
-    def template_line(line)
-      var, value = line.split("=")
-      template = var.gsub(EXPORT_COMMAND, "")
-      is_a_comment = var.strip[0].eql?("#")
-      value.nil? || is_a_comment ? line : "#{var}=#{template}"
+    private
+
+    def is_comment?(line)
+      line.strip.start_with?("#")
+    end
+
+    def var_defined?(line)
+      match = Dotenv::Parser::LINE.match(line)
+      match && match[:key]
+    end
+
+    def line_blank?(line)
+      line.strip.length.zero?
     end
   end
 end

data/lib/dotenv/version.rb

@@ -1,3 +1,3 @@
 module Dotenv
-  VERSION = "2.8.1".freeze
+  VERSION = "3.2.0".freeze
 end

data/lib/dotenv.rb

@@ -1,75 +1,133 @@
+require "dotenv/version"
 require "dotenv/parser"
 require "dotenv/environment"
 require "dotenv/missing_keys"
+require "dotenv/diff"
 
-# The top level Dotenv module. The entrypoint for the application logic.
+# Shim to load environment variables from `.env files into `ENV`.
 module Dotenv
-  class << self
-    attr_accessor :instrumenter
-  end
+  extend self
+
+  # An internal monitor to synchronize access to ENV in multi-threaded environments.
+  SEMAPHORE = Monitor.new
+  private_constant :SEMAPHORE
 
-  module_function
+  attr_accessor :instrumenter
 
-  def load(*filenames)
-    with(*filenames) do |f|
-      ignoring_nonexistent_files do
-        env = Environment.new(f, true)
-        instrument("dotenv.load", env: env) { env.apply }
+  # Loads environment variables from one or more `.env` files. See `#parse` for more details.
+  def load(*filenames, overwrite: false, ignore: true)
+    parse(*filenames, overwrite: overwrite, ignore: ignore) do |env|
+      instrument(:load, env: env) do |payload|
+        update(env, overwrite: overwrite)
       end
     end
   end
 
-  # same as `load`, but raises Errno::ENOENT if any files don't exist
+  # Same as `#load`, but raises Errno::ENOENT if any files don't exist
   def load!(*filenames)
-    with(*filenames) do |f|
-      env = Environment.new(f, true)
-      instrument("dotenv.load", env: env) { env.apply }
-    end
+    load(*filenames, ignore: false)
+  end
+
+  # same as `#load`, but will overwrite existing values in `ENV`
+  def overwrite(*filenames)
+    load(*filenames, overwrite: true)
+  end
+  alias_method :overload, :overwrite
+
+  # same as `#overwrite`, but raises Errno::ENOENT if any files don't exist
+  def overwrite!(*filenames)
+    load(*filenames, overwrite: true, ignore: false)
   end
+  alias_method :overload!, :overwrite!
+
+  # Parses the given files, yielding for each file if a block is given.
+  #
+  # @param filenames [String, Array<String>] Files to parse
+  # @param overwrite [Boolean] Overwrite existing `ENV` values
+  # @param ignore [Boolean] Ignore non-existent files
+  # @param block [Proc] Block to yield for each parsed `Dotenv::Environment`
+  # @return [Hash] parsed key/value pairs
+  def parse(*filenames, overwrite: false, ignore: true, &block)
+    filenames << ".env" if filenames.empty?
+    filenames = filenames.reverse if overwrite
 
-  # same as `load`, but will override existing values in `ENV`
-  def overload(*filenames)
-    with(*filenames) do |f|
-      ignoring_nonexistent_files do
-        env = Environment.new(f, false)
-        instrument("dotenv.overload", env: env) { env.apply! }
+    filenames.reduce({}) do |hash, filename|
+      begin
+        env = Environment.new(File.expand_path(filename), overwrite: overwrite)
+        env = block.call(env) if block
+      rescue Errno::ENOENT, Errno::EISDIR
+        raise unless ignore
       end
+
+      hash.merge! env || {}
     end
   end
 
-  # same as `overload`, but raises Errno::ENOENT if any files don't exist
-  def overload!(*filenames)
-    with(*filenames) do |f|
-      env = Environment.new(f, false)
-      instrument("dotenv.overload", env: env) { env.apply! }
+  # Save the current `ENV` to be restored later
+  def save
+    instrument(:save) do |payload|
+      @diff = payload[:diff] = Dotenv::Diff.new
     end
   end
 
-  # returns a hash of parsed key/value pairs but does not modify ENV
-  def parse(*filenames)
-    with(*filenames) do |f|
-      ignoring_nonexistent_files do
-        Environment.new(f, false)
-      end
+  # Restore `ENV` to a given state
+  #
+  # @param env [Hash] Hash of keys and values to restore, defaults to the last saved state
+  # @param safe [Boolean] Is it safe to modify `ENV`? Defaults to `true` in the main thread, otherwise raises an error.
+  def restore(env = @diff&.a, safe: Thread.current == Thread.main)
+    # No previously saved or provided state to restore
+    return unless env
+
+    diff = Dotenv::Diff.new(b: env)
+    return unless diff.any?
+
+    unless safe
+      raise ThreadError, <<~EOE.tr("\n", " ")
+        Dotenv.restore is not thread safe. Use `Dotenv.modify { }` to update ENV for the duration
+        of the block in a thread safe manner, or call `Dotenv.restore(safe: true)` to ignore
+        this error.
+      EOE
     end
+    instrument(:restore, diff: diff) { ENV.replace(env) }
   end
 
-  # Internal: Helper to expand list of filenames.
+  # Update `ENV` with the given hash of keys and values
   #
-  # Returns a hash of all the loaded environment variables.
-  def with(*filenames)
-    filenames << ".env" if filenames.empty?
-
-    filenames.reduce({}) do |hash, filename|
-      hash.merge!(yield(File.expand_path(filename)) || {})
+  # @param env [Hash] Hash of keys and values to set in `ENV`
+  # @param overwrite [Boolean|:warn] Overwrite existing `ENV` values
+  def update(env = {}, overwrite: false)
+    instrument(:update) do |payload|
+      diff = payload[:diff] = Dotenv::Diff.new do
+        ENV.update(env.transform_keys(&:to_s)) do |key, old_value, new_value|
+          # This block is called when a key exists. Return the new value if overwrite is true.
+          case overwrite
+          when :warn
+            # not printing the value since that could be a secret
+            warn "Warning: dotenv not overwriting ENV[#{key.inspect}]"
+            old_value
+          when true then new_value
+          when false then old_value
+          else raise ArgumentError, "Invalid value for overwrite: #{overwrite.inspect}"
+          end
+        end
+      end
+      diff.env
     end
   end
 
-  def instrument(name, payload = {}, &block)
-    if instrumenter
-      instrumenter.instrument(name, payload, &block)
-    else
-      yield
+  # Modify `ENV` for the block and restore it to its previous state afterwards.
+  #
+  # Note that the block is synchronized to prevent concurrent modifications to `ENV`,
+  # so multiple threads will be executed serially.
+  #
+  # @param env [Hash] Hash of keys and values to set in `ENV`
+  def modify(env = {}, &block)
+    SEMAPHORE.synchronize do
+      diff = Dotenv::Diff.new
+      update(env, overwrite: true)
+      block.call
+    ensure
+      restore(diff.a, safe: true)
     end
   end
 
@@ -79,8 +137,15 @@ module Dotenv
     raise MissingKeys, missing_keys
   end
 
-  def ignoring_nonexistent_files
-    yield
-  rescue Errno::ENOENT
+  private
+
+  def instrument(name, payload = {}, &block)
+    if instrumenter
+      instrumenter.instrument("#{name}.dotenv", payload, &block)
+    else
+      block&.call payload
+    end
   end
 end
+
+require "dotenv/rails" if defined?(Rails::Railtie)