puppet 5.0.0 → 6.0.0

data/lib/puppet/util/execution.rb

@@ -83,7 +83,7 @@ module Puppet::Util::Execution
     end
 
     if failonfail && exitstatus != 0
-      raise Puppet::ExecutionFailure, output
+      raise Puppet::ExecutionFailure, output.to_s
     end
 
     output
@@ -184,22 +184,73 @@ module Puppet::Util::Execution
       Puppet.debug "Executing#{user_log_s}: '#{command_str}'"
     end
 
-    null_file = Puppet.features.microsoft_windows? ? 'NUL' : '/dev/null'
+    null_file = Puppet::Util::Platform.windows? ? 'NUL' : '/dev/null'
 
     begin
       stdin = Puppet::FileSystem.open(options[:stdinfile] || null_file, nil, 'r')
-      stdout = options[:squelch] ? Puppet::FileSystem.open(null_file, nil, 'w') : Puppet::FileSystem::Uniquefile.new('puppet')
+      # On Windows, continue to use the file-based approach to avoid breaking people's existing
+      # manifests. If they use a script that doesn't background cleanly, such as
+      # `start /b ping 127.0.0.1`, we couldn't handle it with pipes as there's no non-blocking
+      # read available.
+      if options[:squelch]
+        stdout = Puppet::FileSystem.open(null_file, nil, 'w')
+      elsif Puppet.features.posix?
+        reader, stdout = IO.pipe
+      else
+        stdout = Puppet::FileSystem::Uniquefile.new('puppet')
+      end
       stderr = options[:combine] ? stdout : Puppet::FileSystem.open(null_file, nil, 'w')
 
       exec_args = [command, options, stdin, stdout, stderr]
+      output = ''
 
+      # We close stdin/stdout/stderr immediately after fork/exec as they're no longer needed by
+      # this process. In most cases they could be closed later, but when `stdout` is the "writer"
+      # pipe we must close it or we'll never reach eof on the `reader` pipe.
       if execution_stub = Puppet::Util::ExecutionStub.current_value
-        return execution_stub.call(*exec_args)
+        child_pid = execution_stub.call(*exec_args)
+        [stdin, stdout, stderr].each {|io| io.close rescue nil}
+        return child_pid
       elsif Puppet.features.posix?
         child_pid = nil
         begin
           child_pid = execute_posix(*exec_args)
-          exit_status = Process.waitpid2(child_pid).last.exitstatus
+          [stdin, stdout, stderr].each {|io| io.close rescue nil}
+          if options[:squelch]
+            exit_status = Process.waitpid2(child_pid).last.exitstatus
+          else
+            # Use non-blocking read to check for data. After each attempt,
+            # check whether the child is done. This is done in case the child
+            # forks and inherits stdout, as happens in `foo &`.
+            until results = Process.waitpid2(child_pid, Process::WNOHANG)
+
+              # If not done, wait for data to read with a timeout
+              # This timeout is selected to keep activity low while waiting on
+              # a long process, while not waiting too long for the pathological
+              # case where stdout is never closed.
+              ready = IO.select([reader], [], [], 0.1)
+              begin
+                output << reader.read_nonblock(4096) if ready
+              rescue Errno::EAGAIN
+              rescue EOFError
+              end
+            end
+
+            # Read any remaining data. Allow for but don't expect EOF.
+            begin
+              loop do
+                output << reader.read_nonblock(4096)
+              end
+            rescue Errno::EAGAIN
+            rescue EOFError
+            end
+
+            # Force to external encoding to preserve prior behavior when reading a file.
+            # Wait until after reading all data so we don't encounter corruption when
+            # reading part of a multi-byte unicode character if default_external is UTF-8.
+            output.force_encoding(Encoding.default_external)
+            exit_status = results.last.exitstatus
+          end
           child_pid = nil
         rescue Timeout::Error => e
           # NOTE: For Ruby 2.1+, an explicit Timeout::Error class has to be
@@ -213,31 +264,33 @@ module Puppet::Util::Execution
 
           raise e
         end
-      elsif Puppet.features.microsoft_windows?
+      elsif Puppet::Util::Platform.windows?
         process_info = execute_windows(*exec_args)
         begin
+          [stdin, stderr].each {|io| io.close rescue nil}
           exit_status = Puppet::Util::Windows::Process.wait_process(process_info.process_handle)
+
+          # read output in if required
+          unless options[:squelch]
+            output = wait_for_output(stdout)
+            Puppet.warning _("Could not get output") unless output
+          end
         ensure
           FFI::WIN32.CloseHandle(process_info.process_handle)
           FFI::WIN32.CloseHandle(process_info.thread_handle)
         end
       end
 
-      [stdin, stdout, stderr].each {|io| io.close rescue nil}
-
-      # read output in if required
-      unless options[:squelch]
-        output = wait_for_output(stdout)
-        Puppet.warning _("Could not get output") unless output
-      end
-
       if options[:failonfail] and exit_status != 0
         raise Puppet::ExecutionFailure, _("Execution of '%{str}' returned %{exit_status}: %{output}") % { str: command_str, exit_status: exit_status, output: output.strip }
       end
     ensure
-      if !options[:squelch] && stdout
-        # if we opened a temp file for stdout, we need to clean it up.
-        stdout.close!
+      # Make sure all handles are closed in case an exception was thrown attempting to execute.
+      [stdin, stdout, stderr].each {|io| io.close rescue nil}
+      if !options[:squelch]
+        # if we opened a pipe, we need to clean it up.
+        reader.close if reader
+        stdout.close! if Puppet::Util::Platform.windows?
       end
     end
 

data/lib/puppet/util/feature.rb

@@ -3,40 +3,56 @@ require 'puppet'
 class Puppet::Util::Feature
   attr_reader :path
 
-  # Create a new feature test.  You have to pass the feature name,
-  # and it must be unique.  You can either provide a block that
-  # will get executed immediately to determine if the feature
-  # is present, or you can pass an option to determine it.
-  # Currently, the only supported option is 'libs' (must be
-  # passed as a symbol), which will make sure that each lib loads
-  # successfully.
-  def add(name, options = {})
+  # Create a new feature test. You have to pass the feature name, and it must be
+  # unique. You can pass a block to determine if the feature is present:
+  #
+  #     Puppet.features.add(:myfeature) do
+  #       # return true or false if feature is available
+  #       # return nil if feature may become available later
+  #     end
+  #
+  # The block should return true if the feature is available, false if it is
+  # not, or nil if the state is unknown. True and false values will be cached. A
+  # nil value will not be cached, and should be used if the feature may become
+  # true in the future.
+  #
+  # Features are often used to detect if a ruby library is installed. To support
+  # that common case, you can pass one or more ruby libraries, and the feature
+  # will be true if all of the libraries load successfully:
+  #
+  #     Puppet.features.add(:myfeature, libs: 'mylib')
+  #     Puppet.features.add(:myfeature, libs: ['mylib', 'myotherlib'])
+  #
+  # If the ruby library is not installed, then the failure is not cached, as
+  # it's assumed puppet may install the gem during catalog application.
+  #
+  # If a feature is defined using `:libs` and a block, then the block is
+  # used and the `:libs` are ignored.
+  #
+  # Puppet evaluates the feature test when the `Puppet.features.myfeature?`
+  # method is called. If the feature test was defined using a block and the
+  # block returns nil, then the feature test will be re-evaluated the next time
+  # `Puppet.features.myfeature?` is called.
+  #
+  # @param [Symbol] name The unique feature name
+  # @param [Hash<Symbol,Array<String>>] options The libraries to load
+  def add(name, options = {}, &block)
     method = name.to_s + "?"
     @results.delete(name)
 
-    if block_given?
-      begin
-        result = yield
-      rescue StandardError,ScriptError => detail
-        warn _("Failed to load feature test for %{name}: %{detail}") % { name: name, detail: detail }
-        result = false
-      end
-      @results[name] = result
-    end
-
     meta_def(method) do
       # we return a cached result if:
-      #  * if a block is given (and we just evaluated it above)
-      #  * if we already have a positive result
-      #  * if we've tested this feature before and it failed, but we're
-      #    configured to always cache
-      if block_given?     ||
-          @results[name]  ||
-          (@results.has_key?(name) && (!Puppet[:always_retry_plugins]))
-        @results[name]
+      #  * if we've tested this feature before
+      #  AND
+      #    * the result was true/false
+      #    OR
+      #    * we're configured to never retry
+      if @results.has_key?(name) &&
+         (!@results[name].nil? || !Puppet[:always_retry_plugins])
+        !!@results[name]
       else
-        @results[name] = test(name, options)
-        @results[name]
+        @results[name] = test(name, options, &block)
+        !!@results[name]
       end
     end
   end
@@ -49,14 +65,14 @@ class Puppet::Util::Feature
   end
 
   def load
-    @loader.loadall
+    @loader.loadall(Puppet.lookup(:current_environment))
   end
 
   def method_missing(method, *args)
     return super unless method.to_s =~ /\?$/
 
     feature = method.to_s.sub(/\?$/, '')
-    @loader.load(feature)
+    @loader.load(feature, Puppet.lookup(:current_environment))
 
     respond_to?(method) && self.send(method)
   end
@@ -64,16 +80,22 @@ class Puppet::Util::Feature
   # Actually test whether the feature is present.  We only want to test when
   # someone asks for the feature, so we don't unnecessarily load
   # files.
-  def test(name, options)
-    return true unless ary = options[:libs]
-    ary = [ary] unless ary.is_a?(Array)
-
-    ary.each do |lib|
-      return false unless load_library(lib, name)
+  def test(name, options, &block)
+    if block_given?
+      begin
+        result = yield
+      rescue StandardError,ScriptError => detail
+        warn _("Failed to load feature test for %{name}: %{detail}") % { name: name, detail: detail }
+        result = nil
+      end
+      @results[name] = result
+      result
+    elsif libs = options[:libs]
+      libs = [libs] unless libs.is_a?(Array)
+      libs.all? { |lib| load_library(lib, name) } ? true : nil
+    else
+      true
     end
-
-    # We loaded all of the required libraries
-    true
   end
 
   private
@@ -86,10 +108,10 @@ class Puppet::Util::Feature
 
     begin
       require lib
+      true
     rescue ScriptError => detail
       Puppet.debug _("Failed to load library '%{lib}' for feature '%{name}': %{detail}") % { lib: lib, name: name, detail: detail }
-      return false
+      false
     end
-    true
   end
 end

data/lib/puppet/util/fileparsing.rb

@@ -24,15 +24,12 @@
 # You could then call 'parser.to_line(hash)' on any of those hashes to generate
 # the text line again.
 
-require 'puppet/util/methodhelper'
-
 module Puppet::Util::FileParsing
   include Puppet::Util
   attr_writer :line_separator, :trailing_separator
 
   class FileRecord
     include Puppet::Util
-    include Puppet::Util::MethodHelper
     attr_accessor :absent, :joiner, :rts, :separator, :rollup, :name, :match, :block_eval
 
     attr_reader :fields, :optional, :type
@@ -48,11 +45,36 @@ module Puppet::Util::FileParsing
       end
     end
 
-    def initialize(type, options = {}, &block)
+    def initialize(type,
+                   absent: nil,
+                   block_eval: nil,
+                   fields: nil,
+                   joiner: nil,
+                   match: nil,
+                   optional: nil,
+                   post_parse: nil,
+                   pre_gen: nil,
+                   rollup: nil,
+                   rts: nil,
+                   separator: nil,
+                   to_line: nil,
+                   &block)
       @type = type.intern
       raise ArgumentError, _("Invalid record type %{record_type}") % { record_type: @type } unless [:record, :text].include?(@type)
 
-      set_options(options)
+      @absent = absent
+      @block_eval = block_eval
+      @joiner = joiner
+      @match = match
+      @rollup = rollup if rollup
+      @rts = rts
+      @separator = separator
+
+      self.fields = fields if fields
+      self.optional = optional if optional
+      self.post_parse = post_parse if post_parse
+      self.pre_gen = pre_gen if pre_gen
+      self.to_line = to_line if to_line
 
       if self.type == :record
         # Now set defaults.
@@ -151,8 +173,7 @@ module Puppet::Util::FileParsing
     if record.respond_to?(:process)
       if ret = record.send(:process, line.dup)
         unless ret.is_a?(Hash)
-          raise Puppet::DevError,
-            "Process record type #{record.name} returned non-hash"
+          raise Puppet::DevError, _("Process record type %{record_name} returned non-hash") % { record_name: record.name }
         end
       else
         return nil
@@ -236,7 +257,7 @@ module Puppet::Util::FileParsing
 
   # Handle parsing a single line.
   def parse_line(line)
-    raise Puppet::DevError, "No record types defined; cannot parse lines" unless records?
+    raise Puppet::DevError, _("No record types defined; cannot parse lines") unless records?
 
     @record_order.each do |record|
       # These are basically either text or record lines.
@@ -247,8 +268,7 @@ module Puppet::Util::FileParsing
           return result
         end
       else
-        raise Puppet::DevError,
-          "Somehow got invalid line type #{record.type}"
+        raise Puppet::DevError, _("Somehow got invalid line type %{record_type}") % { record_type: record.type }
       end
     end
 

data/lib/puppet/util/filetype.rb

@@ -42,7 +42,7 @@ class Puppet::Util::FileType
           else
             return ""
           end
-        rescue Puppet::Error => detail
+        rescue Puppet::Error
           raise
         rescue => detail
           message = _("%{klass} could not read %{path}: %{detail}") % { klass: self.class, path: @path, detail: detail }
@@ -58,7 +58,7 @@ class Puppet::Util::FileType
           val = real_write(text)
           @synced = Time.now
           return val
-        rescue Puppet::Error => detail
+        rescue Puppet::Error
           raise
         rescue => detail
           message = _("%{klass} could not write %{path}: %{detail}") % { klass: self.class, path: @path, detail: detail }

data/lib/puppet/util/http_proxy.rb

@@ -1,5 +1,6 @@
 require 'uri'
 require 'openssl'
+require 'puppet/network/http'
 
 module Puppet::Util::HttpProxy
   def self.proxy(uri)
@@ -156,8 +157,10 @@ module Puppet::Util::HttpProxy
     proxy
   end
 
-  # Retrieve a document through HTTP(s), following redirects if necessary.
-  # 
+  # Retrieve a document through HTTP(s), following redirects if necessary. The
+  # returned response body may be compressed, and it is the caller's
+  # responsibility to decompress it based on the 'content-encoding' header.
+  #
   # Based on the the client implementation in the HTTP pool.
   #
   # @see Puppet::Network::HTTP::Connection#request_with_redirects
@@ -172,7 +175,13 @@ module Puppet::Util::HttpProxy
 
     0.upto(redirect_limit) do |redirection|
       proxy = get_http_object(current_uri)
-      response = proxy.send(:head, current_uri.path)
+
+      headers = { 'Accept' => '*/*', 'User-Agent' => Puppet[:http_user_agent] }
+      if Puppet.features.zlib?
+        headers.merge!({"Accept-Encoding" => Puppet::Network::HTTP::Compression::ACCEPT_ENCODING})
+      end
+
+      response = proxy.send(:head, current_uri.path, headers)
 
       if [301, 302, 307].include?(response.code.to_i)
         # handle the redirection
@@ -180,11 +189,12 @@ module Puppet::Util::HttpProxy
         next
       end
 
-      if block_given?
-        headers = {'Accept' => 'binary', 'accept-encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3'}
-        response = proxy.send("request_#{method}".to_sym, current_uri.path, headers, &block)
-      else
-        response = proxy.send(method, current_uri.path)
+      if method != :head
+        if block_given?
+          response = proxy.send("request_#{method}".to_sym, current_uri.path, headers, &block)
+        else
+          response = proxy.send(method, current_uri.path, headers)
+        end
       end
 
       Puppet.debug("HTTP #{method.to_s.upcase} request to #{current_uri} returned #{response.code} #{response.message}")

data/lib/puppet/util/inifile.rb

@@ -148,7 +148,7 @@ module Puppet::Util::IniConfig
     )
     INI_CONTINUATION = /^[ \t\r\n\f]/
     INI_SECTION_NAME = /^\[([^\]]+)\]/
-    INI_PROPERTY     = /^\s*([^\s=]+)\s*\=(.*)$/
+    INI_PROPERTY     = /^\s*([^\s=]+)\s*\=\s*(.*)$/
 
     # @api private
     def parse(text)
@@ -177,10 +177,9 @@ module Puppet::Util::IniConfig
           section = add_section(section_name)
           optname = nil
         elsif (match = l.match(INI_PROPERTY))
-          # We allow space around the keys, but not the values
-          # For the values, we don't know if space is significant
+          # the regex strips leading white space from the value, and here we strip the trailing white space as well
           key = match[1]
-          val = match[2]
+          val = match[2].rstrip
 
           if section.nil?
             raise IniParseError.new(_("Property with key %{key} outside of a section") % { key: key.inspect })

data/lib/puppet/util/instance_loader.rb

@@ -12,12 +12,12 @@ module Puppet::Util::InstanceLoader
   end
 
   # Define a new type of autoloading.
-  def instance_load(type, path, options = {})
+  def instance_load(type, path)
     @autoloaders ||= {}
     @instances ||= {}
     type = type.intern
     @instances[type] = {}
-    @autoloaders[type] = Puppet::Util::Autoload.new(self, path, options)
+    @autoloaders[type] = Puppet::Util::Autoload.new(self, path)
 
     # Now define our new simple methods
     unless respond_to?(type)
@@ -32,24 +32,6 @@ module Puppet::Util::InstanceLoader
     @instances[type].keys
   end
 
-  # Collect the docs for all of our instances.
-  def instance_docs(type)
-    docs = ""
-
-    # Load all instances.
-    instance_loader(type).loadall
-
-    # Use this method so they all get loaded
-    loaded_instances(type).sort { |a,b| a.to_s <=> b.to_s }.each do |name|
-      mod = self.loaded_instance(name)
-      docs << "#{name}\n#{"-" * name.to_s.length}\n"
-
-      docs << Puppet::Util::Docs.scrub(mod.doc) << "\n\n"
-    end
-
-    docs
-  end
-
   # Return the instance hash for our type.
   def instance_hash(type)
     @instances[type.intern]
@@ -65,11 +47,9 @@ module Puppet::Util::InstanceLoader
     name = name.intern
     return nil unless instances = instance_hash(type)
     unless instances.include? name
-      if instance_loader(type).load(name)
+      if instance_loader(type).load(name, Puppet.lookup(:current_environment))
         unless instances.include? name
-          Puppet.warning(
-            _("Loaded %{type} file for %{name} but %{type} was not defined") % { type: type, name: name }
-          )
+          Puppet.warning(_("Loaded %{type} file for %{name} but %{type} was not defined") % { type: type, name: name })
           return nil
         end
       else

data/lib/puppet/util/json.rb

@@ -0,0 +1,69 @@
+module Puppet::Util
+  module Json
+    class ParseError < StandardError
+      attr_reader :cause, :data
+
+      def self.build(original_exception, data)
+        new(original_exception.message).tap do |exception|
+          exception.instance_eval do
+            @cause = original_exception
+            set_backtrace original_exception.backtrace
+            @data = data
+          end
+        end
+      end
+    end
+
+    begin
+      require 'multi_json'
+      # Force backend detection before attempting to use the library
+      # or load any other JSON libraries
+      MultiJson.default_adapter
+
+      # Preserve core type monkey-patching done by the built-in JSON gem
+      require 'json'
+    rescue LoadError
+      require 'json'
+    end
+
+    # These methods do similar processing to the fallback implemented by MultiJson
+    # when using the built-in JSON backend, to ensure consistent behavior
+    # whether or not MultiJson can be loaded.
+    def self.load(string, options = {})
+      if defined? MultiJson
+        begin
+          # This ensures that JrJackson will parse very large or very small
+          # numbers as floats rather than BigDecimals, which are serialized as
+          # strings by the built-in JSON gem and therefore can cause schema errors,
+          # for example, when we are rendering reports to JSON using `to_pson` in
+          # PuppetDB.
+          if MultiJson.adapter.name == "MultiJson::Adapters::JrJackson"
+            options[:use_bigdecimal] = false
+          end
+
+          MultiJson.load(string, options)
+        rescue MultiJson::ParseError => e
+          raise Puppet::Util::Json::ParseError.build(e, string)
+        end
+      else
+        begin
+          string = string.read if string.respond_to?(:read)
+
+          options[:symbolize_names] = true if options.delete(:symbolize_keys)
+          ::JSON.parse(string, options)
+        rescue JSON::ParserError => e
+          raise Puppet::Util::Json::ParseError.build(e, string)
+        end
+      end
+    end
+
+    def self.dump(object, options = {})
+      if defined? MultiJson
+        MultiJson.dump(object, options)
+      else
+        options.merge!(::JSON::PRETTY_STATE_PROTOTYPE.to_h) if options.delete(:pretty)
+        object.to_json(options)
+      end
+    end
+  end
+end

data/lib/puppet/util/json_lockfile.rb

@@ -23,7 +23,7 @@ class Puppet::Util::JsonLockfile < Puppet::Util::Lockfile
   def lock(lock_data = nil)
     return false if locked?
 
-    super(lock_data.to_json)
+    super(Puppet::Util::Json.dump(lock_data))
   end
 
   # Retrieve the (optional) lock data that was specified at the time the file
@@ -35,8 +35,8 @@ class Puppet::Util::JsonLockfile < Puppet::Util::Lockfile
     return nil unless file_locked?
     file_contents = super
     return nil if file_contents.nil? or file_contents.empty?
-    JSON.parse(file_contents)
-  rescue JSON::ParserError => e
+    Puppet::Util::Json.load(file_contents)
+  rescue Puppet::Util::Json::ParseError
     Puppet.warning _("Unable to read lockfile data from %{path}: not in JSON") % { path: @file_path }
     nil
   end

data/lib/puppet/util/ldap/connection.rb

@@ -1,9 +1,6 @@
 require 'puppet/util/ldap'
-require 'puppet/util/methodhelper'
 
 class Puppet::Util::Ldap::Connection
-  include Puppet::Util::MethodHelper
-
   attr_accessor :host, :port, :user, :password, :reset, :ssl
 
   attr_reader :connection
@@ -34,12 +31,15 @@ class Puppet::Util::Ldap::Connection
     connection.unbind if connection.bound?
   end
 
-  def initialize(host, port, options = {})
+  def initialize(host, port, user: nil, password: nil, reset: nil, ssl: nil)
     raise Puppet::Error, _("Could not set up LDAP Connection: Missing ruby/ldap libraries") unless Puppet.features.ldap?
 
-    @host, @port = host, port
-
-    set_options(options)
+    @host = host
+    @port = port
+    @user = user
+    @password = password
+    @reset = reset
+    @ssl = ssl
   end
 
   # Create a per-connection unique name.

data/lib/puppet/util/ldap/manager.rb

@@ -44,7 +44,8 @@ class Puppet::Util::Ldap::Manager
   # Open, yield, and close the connection.  Cannot be left
   # open, at this point.
   def connect
-    raise ArgumentError, "You must pass a block to #connect" unless block_given?
+    #TRANSLATORS '#connect' is a method name and and should not be translated, 'block' refers to a Ruby code block
+    raise ArgumentError, _("You must pass a block to #connect") unless block_given?
 
     unless @connection
       if Puppet[:ldaptls]
@@ -85,7 +86,8 @@ class Puppet::Util::Ldap::Manager
 
   # Convert an ldap-style entry hash to a provider-style hash.
   def entry2provider(entry)
-    raise ArgumentError, "Could not get dn from ldap entry" unless entry["dn"]
+    #TRANSLATOR 'dn' refers to a 'distinguished name' in LDAP (Lightweight Directory Access Protocol) and they should not be translated
+    raise ArgumentError, _("Could not get dn from ldap entry") unless entry["dn"]
 
     # DN is always a single-entry array.  Strip off the bits before the
     # first comma, then the bits after the remaining equal sign.  This is the
@@ -137,7 +139,8 @@ class Puppet::Util::Ldap::Manager
 
       if generator.source
         unless value = values[generator.source]
-          raise ArgumentError, "#{generator.source} must be defined to generate #{generator.name}"
+          raise ArgumentError, _("%{source} must be defined to generate %{name}") %
+              { source: generator.source, name: generator.name }
         end
         result = generator.generate(value)
       else