puppet 5.3.4 → 6.29.0

data/lib/puppet/util/fileparsing.rb

@@ -24,15 +24,12 @@
 # You could then call 'parser.to_line(hash)' on any of those hashes to generate
 # the text line again.
 
-require 'puppet/util/methodhelper'
-
 module Puppet::Util::FileParsing
   include Puppet::Util
   attr_writer :line_separator, :trailing_separator
 
   class FileRecord
     include Puppet::Util
-    include Puppet::Util::MethodHelper
     attr_accessor :absent, :joiner, :rts, :separator, :rollup, :name, :match, :block_eval
 
     attr_reader :fields, :optional, :type
@@ -48,11 +45,36 @@ module Puppet::Util::FileParsing
       end
     end
 
-    def initialize(type, options = {}, &block)
+    def initialize(type,
+                   absent: nil,
+                   block_eval: nil,
+                   fields: nil,
+                   joiner: nil,
+                   match: nil,
+                   optional: nil,
+                   post_parse: nil,
+                   pre_gen: nil,
+                   rollup: nil,
+                   rts: nil,
+                   separator: nil,
+                   to_line: nil,
+                   &block)
       @type = type.intern
       raise ArgumentError, _("Invalid record type %{record_type}") % { record_type: @type } unless [:record, :text].include?(@type)
 
-      set_options(options)
+      @absent = absent
+      @block_eval = block_eval
+      @joiner = joiner
+      @match = match
+      @rollup = rollup if rollup
+      @rts = rts
+      @separator = separator
+
+      self.fields = fields if fields
+      self.optional = optional if optional
+      self.post_parse = post_parse if post_parse
+      self.pre_gen = pre_gen if pre_gen
+      self.to_line = to_line if to_line
 
       if self.type == :record
         # Now set defaults.
@@ -128,7 +150,8 @@ module Puppet::Util::FileParsing
   end
 
   def fields(type)
-    if record = record_type(type)
+    record = record_type(type)
+    if record
       record.fields.dup
     else
       nil
@@ -149,51 +172,56 @@ module Puppet::Util::FileParsing
   def handle_record_line(line, record)
     ret = nil
     if record.respond_to?(:process)
-      if ret = record.send(:process, line.dup)
+      ret = record.send(:process, line.dup)
+      if ret
         unless ret.is_a?(Hash)
           raise Puppet::DevError, _("Process record type %{record_name} returned non-hash") % { record_name: record.name }
         end
       else
         return nil
       end
-    elsif regex = record.match
-      # In this case, we try to match the whole line and then use the
-      # match captures to get our fields.
-      if match = regex.match(line)
-        ret = {}
-        record.fields.zip(match.captures).each do |field, value|
-          if value == record.absent
-            ret[field] = :absent
-          else
-            ret[field] = value
+    else
+      regex = record.match
+      if regex
+        # In this case, we try to match the whole line and then use the
+        # match captures to get our fields.
+        match = regex.match(line)
+        if match
+          ret = {}
+          record.fields.zip(match.captures).each do |field, value|
+            if value == record.absent
+              ret[field] = :absent
+            else
+              ret[field] = value
+            end
           end
+        else
+          nil
         end
       else
-        nil
-      end
-    else
-      ret = {}
-      sep = record.separator
+        ret = {}
+        sep = record.separator
 
-      # String "helpfully" replaces ' ' with /\s+/ in splitting, so we
-      # have to work around it.
-      if sep == " "
-        sep = / /
-      end
-      line_fields = line.split(sep)
-      record.fields.each do |param|
-        value = line_fields.shift
-        if value and value != record.absent
-          ret[param] = value
-        else
-          ret[param] = :absent
+        # String "helpfully" replaces ' ' with /\s+/ in splitting, so we
+        # have to work around it.
+        if sep == " "
+          sep = / /
+        end
+        line_fields = line.split(sep)
+        record.fields.each do |param|
+          value = line_fields.shift
+          if value and value != record.absent
+            ret[param] = value
+          else
+            ret[param] = :absent
+          end
         end
-      end
 
-      if record.rollup and ! line_fields.empty?
-        last_field = record.fields[-1]
-        val = ([ret[last_field]] + line_fields).join(record.joiner)
-        ret[last_field] = val
+        if record.rollup and ! line_fields.empty?
+          last_field = record.fields[-1]
+          val = ([ret[last_field]] + line_fields).join(record.joiner)
+          ret[last_field] = val
+        end
       end
     end
 
@@ -223,7 +251,8 @@ module Puppet::Util::FileParsing
     count = 1
     lines(text).collect do |line|
       count += 1
-      if val = parse_line(line)
+      val = parse_line(line)
+      if val
         val
       else
         error = Puppet::ResourceError.new(_("Could not parse line %{line}") % { line: line.inspect })
@@ -241,7 +270,8 @@ module Puppet::Util::FileParsing
       # These are basically either text or record lines.
       method = "handle_#{record.type}_line"
       if respond_to?(method)
-        if result = send(method, line, record)
+        result = send(method, line, record)
+        if result
           record.send(:post_parse, result) if record.respond_to?(:post_parse)
           return result
         end
@@ -270,7 +300,7 @@ module Puppet::Util::FileParsing
   def record_line(name, options, &block)
     raise ArgumentError, _("Must include a list of fields") unless options.include?(:fields)
 
-    record = FileRecord.new(:record, options, &block)
+    record = FileRecord.new(:record, **options, &block)
     record.name = name.intern
 
     new_line_type(record)
@@ -285,7 +315,7 @@ module Puppet::Util::FileParsing
   def text_line(name, options, &block)
     raise ArgumentError, _("You must provide a :match regex for text lines") unless options.include?(:match)
 
-    record = FileRecord.new(:text, options, &block)
+    record = FileRecord.new(:text, **options, &block)
     record.name = name.intern
 
     new_line_type(record)
@@ -302,7 +332,8 @@ module Puppet::Util::FileParsing
 
   # Convert our parsed record into a text record.
   def to_line(details)
-    unless record = record_type(details[:record_type])
+    record = record_type(details[:record_type])
+    unless record
       raise ArgumentError, _("Invalid record type %{record_type}") % { record_type: details[:record_type].inspect }
     end
 
@@ -318,7 +349,8 @@ module Puppet::Util::FileParsing
 
       line = record.join(details)
 
-      if regex = record.rts
+      regex = record.rts
+      if regex
         # If they say true, then use whitespace; else, use their regex.
         if regex == true
           regex = /\s+$/
@@ -341,7 +373,8 @@ module Puppet::Util::FileParsing
 
   def valid_attr?(type, attr)
     type = type.intern
-    if record = record_type(type) and record.fields.include?(attr.intern)
+    record = record_type(type)
+    if record && record.fields.include?(attr.intern)
       return true
     else
       if attr.intern == :ensure

data/lib/puppet/util/filetype.rb

@@ -42,7 +42,7 @@ class Puppet::Util::FileType
           else
             return ""
           end
-        rescue Puppet::Error => detail
+        rescue Puppet::Error
           raise
         rescue => detail
           message = _("%{klass} could not read %{path}: %{detail}") % { klass: self.class, path: @path, detail: detail }
@@ -58,7 +58,7 @@ class Puppet::Util::FileType
           val = real_write(text)
           @synced = Time.now
           return val
-        rescue Puppet::Error => detail
+        rescue Puppet::Error
           raise
         rescue => detail
           message = _("%{klass} could not write %{path}: %{detail}") % { klass: self.class, path: @path, detail: detail }
@@ -88,7 +88,8 @@ class Puppet::Util::FileType
   # to the target user if the target user and the current user are not
   # the same
   def cronargs
-    if uid = Puppet::Util.uid(@path) and uid == Puppet::Util::SUIDManager.uid
+    uid = Puppet::Util.uid(@path)
+    if uid && uid == Puppet::Util::SUIDManager.uid
       {:failonfail => true, :combine => true}
     else
       {:failonfail => true, :combine => true, :uid => @path}
@@ -166,6 +167,12 @@ class Puppet::Util::FileType
   end
 
   # Handle Linux-style cron tabs.
+  #
+  # TODO: We can possibly eliminate the "-u <username>" option in cmdbase
+  # by just running crontab under <username>'s uid (like we do for suntab
+  # and aixtab). It may be worth investigating this alternative
+  # implementation in the future. This way, we can refactor all three of
+  # our cron file types into a common crontab file type.
   newfiletype(:crontab) do
     def initialize(user)
       self.path = user
@@ -185,21 +192,47 @@ class Puppet::Util::FileType
 
     # Read a specific @path's cron tab.
     def read
-      %x{#{cmdbase} -l 2>/dev/null}
+      unless Puppet::Util.uid(@path)
+        Puppet.debug _("The %{path} user does not exist. Treating their crontab file as empty in case Puppet creates them in the middle of the run.") % { path: @path }
+
+        return ""
+      end
+
+      Puppet::Util::Execution.execute("#{cmdbase} -l", failonfail: true, combine: true)
+    rescue => detail
+      case detail.to_s
+      when /no crontab for/
+        return ""
+      when /are not allowed to/
+        Puppet.debug _("The %{path} user is not authorized to use cron. Their crontab file is treated as empty in case Puppet authorizes them in the middle of the run (by, for example, modifying the cron.deny or cron.allow files).") % { path: @path }
+
+        return ""
+      else
+        raise FileReadError, _("Could not read crontab for %{path}: %{detail}") % { path: @path, detail: detail }, detail.backtrace
+      end
     end
 
     # Remove a specific @path's cron tab.
     def remove
-      if %w{Darwin FreeBSD DragonFly}.include?(Facter.value("operatingsystem"))
-        %x{/bin/echo yes | #{cmdbase} -r 2>/dev/null}
-      else
-        %x{#{cmdbase} -r 2>/dev/null}
+      cmd = "#{cmdbase} -r"
+      if %w{Darwin FreeBSD DragonFly}.include?(Puppet.runtime[:facter].value("operatingsystem"))
+        cmd = "/bin/echo yes | #{cmd}"
       end
+
+      Puppet::Util::Execution.execute(cmd, failonfail: true, combine: true)
     end
 
     # Overwrite a specific @path's cron tab; must be passed the @path name
     # and the text with which to create the cron tab.
+    #
+    # TODO: We should refactor this at some point to make it identical to the
+    # :aixtab and :suntab's write methods so that, at the very least, the pipe
+    # is not created and the crontab command's errors are not swallowed.
     def write(text)
+      unless Puppet::Util.uid(@path)
+        raise Puppet::Error, _("Cannot write the %{path} user's crontab: The user does not exist") % { path: @path }
+      end
+
       # this file is managed by the OS and should be using system encoding
       IO.popen("#{cmdbase()} -", "w", :encoding => Encoding.default_external) { |p|
         p.print text
@@ -211,7 +244,7 @@ class Puppet::Util::FileType
     # Only add the -u flag when the @path is different.  Fedora apparently
     # does not think I should be allowed to set the @path to my own user name
     def cmdbase
-      if @uid == Puppet::Util::SUIDManager.uid || Facter.value(:operatingsystem) == "HP-UX"
+      if @uid == Puppet::Util::SUIDManager.uid || Puppet.runtime[:facter].value(:operatingsystem) == "HP-UX"
         return "crontab"
       else
         return "crontab -u #{@path}"
@@ -224,13 +257,21 @@ class Puppet::Util::FileType
   newfiletype(:suntab) do
     # Read a specific @path's cron tab.
     def read
+      unless Puppet::Util.uid(@path)
+        Puppet.debug _("The %{path} user does not exist. Treating their crontab file as empty in case Puppet creates them in the middle of the run.") % { path: @path }
+
+        return ""
+      end
+
       Puppet::Util::Execution.execute(%w{crontab -l}, cronargs)
     rescue => detail
       case detail.to_s
       when /can't open your crontab/
         return ""
       when /you are not authorized to use cron/
-        raise FileReadError, _("User %{path} not authorized to use cron") % { path: @path }, detail.backtrace
+        Puppet.debug _("The %{path} user is not authorized to use cron. Their crontab file is treated as empty in case Puppet authorizes them in the middle of the run (by, for example, modifying the cron.deny or cron.allow files).") % { path: @path }
+
+        return ""
       else
         raise FileReadError, _("Could not read crontab for %{path}: %{detail}") % { path: @path, detail: detail }, detail.backtrace
       end
@@ -267,13 +308,21 @@ class Puppet::Util::FileType
   newfiletype(:aixtab) do
     # Read a specific @path's cron tab.
     def read
+      unless Puppet::Util.uid(@path)
+        Puppet.debug _("The %{path} user does not exist. Treating their crontab file as empty in case Puppet creates them in the middle of the run.") % { path: @path }
+
+        return ""
+      end
+
       Puppet::Util::Execution.execute(%w{crontab -l}, cronargs)
     rescue => detail
       case detail.to_s
-      when /Cannot open a file in the .* directory/
+      when /open.*in.*directory/
+        return ""
+      when /not.*authorized.*cron/
+        Puppet.debug _("The %{path} user is not authorized to use cron. Their crontab file is treated as empty in case Puppet authorizes them in the middle of the run (by, for example, modifying the cron.deny or cron.allow files).") % { path: @path }
+
         return ""
-      when /You are not authorized to use the cron command/
-        raise FileReadError, _("User %{path} not authorized to use cron") % { path: @path }, detail.backtrace
       else
         raise FileReadError, _("Could not read crontab for %{path}: %{detail}") % { path: @path, detail: detail }, detail.backtrace
       end

data/lib/puppet/util/http_proxy.rb

@@ -1,16 +1,18 @@
 require 'uri'
-require 'openssl'
-require 'puppet/network/http'
+require 'puppet/ssl/openssl_loader'
+require 'puppet/http'
 
 module Puppet::Util::HttpProxy
   def self.proxy(uri)
-    if self.no_proxy?(uri)
-      proxy_class = Net::HTTP::Proxy(nil)
+    if http_proxy_host && !no_proxy?(uri)
+      Net::HTTP.new(uri.host, uri.port, self.http_proxy_host, self.http_proxy_port, self.http_proxy_user, self.http_proxy_password)
     else
-      proxy_class = Net::HTTP::Proxy(self.http_proxy_host, self.http_proxy_port, self.http_proxy_user, self.http_proxy_password)
+      http = Net::HTTP.new(uri.host, uri.port, nil, nil, nil, nil)
+      # Net::HTTP defaults the proxy port even though we said not to
+      # use one. Set it to nil so caller is not surprised
+      http.proxy_port = nil
+      http
     end
-
-    return proxy_class.new(uri.host, uri.port)
   end
 
   def self.http_proxy_env
@@ -33,7 +35,8 @@ module Puppet::Util::HttpProxy
   #   .example.com
   # We'll accommodate both here.
   def self.no_proxy?(dest)
-    unless no_proxy_env = ENV["no_proxy"] || ENV["NO_PROXY"]
+    no_proxy = self.no_proxy
+    unless no_proxy
       return false
     end
 
@@ -45,18 +48,10 @@ module Puppet::Util::HttpProxy
       end
     end
 
-    no_proxy_env.split(/\s*,\s*/).each do |d|
+    no_proxy.split(/\s*,\s*/).each do |d|
       host, port = d.split(':')
       host = Regexp.escape(host).gsub('\*', '.*')
 
-      #If the host of this no_proxy value starts with '.', this entry is
-      #a domain level entry. Don't pin the regex to the beginning of the entry.
-      #If it does not start with a '.' then it is a host specific entry and
-      #should be matched to the destination starting at the beginning.
-      unless host =~ /^\\\./
-        host = "^#{host}"
-      end
-
       #If this no_proxy entry specifies a port, we want to match it against
       #the destination port.  Otherwise just match hosts.
       if port
@@ -127,6 +122,20 @@ module Puppet::Util::HttpProxy
     return Puppet.settings[:http_proxy_password]
   end
 
+  def self.no_proxy
+    no_proxy_env = ENV["no_proxy"] || ENV["NO_PROXY"]
+
+    if no_proxy_env
+      return no_proxy_env
+    end
+
+    if Puppet.settings[:no_proxy] == 'none'
+      return nil
+    end
+
+    return Puppet.settings[:no_proxy]
+  end
+
   # Return a Net::HTTP::Proxy object.
   #
   # This method optionally configures SSL correctly if the URI scheme is
@@ -157,8 +166,10 @@ module Puppet::Util::HttpProxy
     proxy
   end
 
-  # Retrieve a document through HTTP(s), following redirects if necessary.
-  # 
+  # Retrieve a document through HTTP(s), following redirects if necessary. The
+  # returned response body may be compressed, and it is the caller's
+  # responsibility to decompress it based on the 'content-encoding' header.
+  #
   # Based on the the client implementation in the HTTP pool.
   #
   # @see Puppet::Network::HTTP::Connection#request_with_redirects
@@ -176,10 +187,11 @@ module Puppet::Util::HttpProxy
 
       headers = { 'Accept' => '*/*', 'User-Agent' => Puppet[:http_user_agent] }
       if Puppet.features.zlib?
-        headers.merge!({"Accept-Encoding" => Puppet::Network::HTTP::Compression::ACCEPT_ENCODING})
+        headers["Accept-Encoding"] = Puppet::HTTP::ACCEPT_ENCODING
       end
 
-      response = proxy.send(:head, current_uri.path, headers)
+      response = proxy.send(:head, current_uri, headers)
+      Puppet.debug("HTTP HEAD request to #{current_uri} returned #{response.code} #{response.message}")
 
       if [301, 302, 307].include?(response.code.to_i)
         # handle the redirection
@@ -189,13 +201,13 @@ module Puppet::Util::HttpProxy
 
       if method != :head
         if block_given?
-          response = proxy.send("request_#{method}".to_sym, current_uri.path, headers, &block)
+          response = proxy.send("request_#{method}".to_sym, current_uri, headers, &block)
         else
-          response = proxy.send(method, current_uri.path, headers)
+          response = proxy.send(method, current_uri, headers)
         end
-      end
 
-      Puppet.debug("HTTP #{method.to_s.upcase} request to #{current_uri} returned #{response.code} #{response.message}")
+        Puppet.debug("HTTP #{method.to_s.upcase} request to #{current_uri} returned #{response.code} #{response.message}")
+      end
 
       return response
     end

data/lib/puppet/util/inifile.rb

@@ -148,7 +148,7 @@ module Puppet::Util::IniConfig
     )
     INI_CONTINUATION = /^[ \t\r\n\f]/
     INI_SECTION_NAME = /^\[([^\]]+)\]/
-    INI_PROPERTY     = /^\s*([^\s=]+)\s*\=(.*)$/
+    INI_PROPERTY     = /^\s*([^\s=]+)\s*\=\s*(.*)$/
 
     # @api private
     def parse(text)
@@ -177,10 +177,9 @@ module Puppet::Util::IniConfig
           section = add_section(section_name)
           optname = nil
         elsif (match = l.match(INI_PROPERTY))
-          # We allow space around the keys, but not the values
-          # For the values, we don't know if space is significant
+          # the regex strips leading white space from the value, and here we strip the trailing white space as well
           key = match[1]
-          val = match[2]
+          val = match[2].rstrip
 
           if section.nil?
             raise IniParseError.new(_("Property with key %{key} outside of a section") % { key: key.inspect })

data/lib/puppet/util/instance_loader.rb

@@ -1,5 +1,6 @@
 require 'puppet/util/autoload'
 require 'puppet/util'
+require 'puppet/concurrent/lock'
 
 # A module that can easily autoload things for us.  Uses an instance
 # of Puppet::Util::Autoload
@@ -12,12 +13,13 @@ module Puppet::Util::InstanceLoader
   end
 
   # Define a new type of autoloading.
-  def instance_load(type, path, options = {})
+  def instance_load(type, path)
     @autoloaders ||= {}
     @instances ||= {}
     type = type.intern
     @instances[type] = {}
-    @autoloaders[type] = Puppet::Util::Autoload.new(self, path, options)
+    @autoloaders[type] = Puppet::Util::Autoload.new(self, path)
+    @instance_loader_lock = Puppet::Concurrent::Lock.new
 
     # Now define our new simple methods
     unless respond_to?(type)
@@ -32,24 +34,6 @@ module Puppet::Util::InstanceLoader
     @instances[type].keys
   end
 
-  # Collect the docs for all of our instances.
-  def instance_docs(type)
-    docs = ""
-
-    # Load all instances.
-    instance_loader(type).loadall
-
-    # Use this method so they all get loaded
-    loaded_instances(type).sort { |a,b| a.to_s <=> b.to_s }.each do |name|
-      mod = self.loaded_instance(name)
-      docs << "#{name}\n#{"-" * name.to_s.length}\n"
-
-      docs << Puppet::Util::Docs.scrub(mod.doc) << "\n\n"
-    end
-
-    docs
-  end
-
   # Return the instance hash for our type.
   def instance_hash(type)
     @instances[type.intern]
@@ -62,18 +46,21 @@ module Puppet::Util::InstanceLoader
 
   # Retrieve an already-loaded instance, or attempt to load our instance.
   def loaded_instance(type, name)
-    name = name.intern
-    return nil unless instances = instance_hash(type)
-    unless instances.include? name
-      if instance_loader(type).load(name)
-        unless instances.include? name
-          Puppet.warning(_("Loaded %{type} file for %{name} but %{type} was not defined") % { type: type, name: name })
+    @instance_loader_lock.synchronize do
+      name = name.intern
+      instances = instance_hash(type)
+      return nil unless instances
+      unless instances.include? name
+        if instance_loader(type).load(name, Puppet.lookup(:current_environment))
+          unless instances.include? name
+            Puppet.warning(_("Loaded %{type} file for %{name} but %{type} was not defined") % { type: type, name: name })
+            return nil
+          end
+        else
           return nil
         end
-      else
-        return nil
       end
+      instances[name]
     end
-    instances[name]
   end
 end

data/lib/puppet/util/json.rb

@@ -0,0 +1,89 @@
+module Puppet::Util
+  module Json
+    class ParseError < StandardError
+      attr_reader :cause, :data
+
+      def self.build(original_exception, data)
+        new(original_exception.message).tap do |exception|
+          exception.instance_eval do
+            @cause = original_exception
+            set_backtrace original_exception.backtrace
+            @data = data
+          end
+        end
+      end
+    end
+
+    begin
+      require 'multi_json'
+      # Force backend detection before attempting to use the library
+      # or load any other JSON libraries
+      MultiJson.default_adapter
+
+      # Preserve core type monkey-patching done by the built-in JSON gem
+      require 'json'
+    rescue LoadError
+      require 'json'
+    end
+
+    # Load the content from a file as JSON if
+    # contents are in valid format. This method does not
+    # raise error but returns `nil` when invalid file is
+    # given.
+    def self.load_file_if_valid(filename, options = {})
+      load_file(filename, options)
+    rescue Puppet::Util::Json::ParseError, ArgumentError, Errno::ENOENT => detail
+      Puppet.debug("Could not retrieve JSON content from '#{filename}': #{detail.message}")
+      nil
+    end
+
+    # Load the content from a file as JSON.
+    def self.load_file(filename, options = {})
+      json = Puppet::FileSystem.read(filename, :encoding => 'utf-8')
+      load(json, options)
+    end
+
+    # These methods do similar processing to the fallback implemented by MultiJson
+    # when using the built-in JSON backend, to ensure consistent behavior
+    # whether or not MultiJson can be loaded.
+    def self.load(string, options = {})
+      if defined? MultiJson
+        begin
+          # This ensures that JrJackson will parse very large or very small
+          # numbers as floats rather than BigDecimals, which are serialized as
+          # strings by the built-in JSON gem and therefore can cause schema errors,
+          # for example, when we are rendering reports to JSON using `to_pson` in
+          # PuppetDB.
+          if MultiJson.adapter.name == "MultiJson::Adapters::JrJackson"
+            options[:use_bigdecimal] = false
+          end
+
+          MultiJson.load(string, options)
+        rescue MultiJson::ParseError => e
+          raise Puppet::Util::Json::ParseError.build(e, string)
+        end
+      else
+        begin
+          string = string.read if string.respond_to?(:read)
+
+          options[:symbolize_names] = true if options.delete(:symbolize_keys)
+          ::JSON.parse(string, options)
+        rescue JSON::ParserError => e
+          raise Puppet::Util::Json::ParseError.build(e, string)
+        end
+      end
+    end
+
+    def self.dump(object, options = {})
+      if defined? MultiJson
+        MultiJson.dump(object, options)
+      elsif options.is_a?(JSON::State)
+        # we're being called recursively
+        object.to_json(options)
+      else
+        options.merge!(::JSON::PRETTY_STATE_PROTOTYPE.to_h) if options.delete(:pretty)
+        object.to_json(options)
+      end
+    end
+  end
+end