puppet 4.0.0 → 4.9.1

data/lib/puppet/util/checksums.rb

@@ -1,5 +1,6 @@
 require 'digest/md5'
 require 'digest/sha1'
+require 'time'
 
 # A stand-alone module for calculating checksums
 # in a generic way.
@@ -41,8 +42,8 @@ module Puppet::Util::Checksums
     Digest::SHA256.hexdigest(content)
   end
 
-  def sha256lite(content)
-    sha256(content[0..511])
+  def sha256?(string)
+    string =~ /^\h{64}$/
   end
 
   def sha256_file(filename, lite = false)
@@ -52,10 +53,6 @@ module Puppet::Util::Checksums
     checksum_file(digest, filename,  lite)
   end
 
-  def sha256lite_file(filename)
-    sha256_file(filename, true)
-  end
-
   def sha256_stream(lite = false, &block)
     require 'digest/sha2'
     digest = Digest::SHA256.new
@@ -66,6 +63,18 @@ module Puppet::Util::Checksums
     64
   end
 
+  def sha256lite(content)
+    sha256(content[0..511])
+  end
+
+  def sha256lite?(string)
+    sha256?(string)
+  end
+
+  def sha256lite_file(filename)
+    sha256_file(filename, true)
+  end
+
   def sha256lite_stream(&block)
     sha256_stream(true, &block)
   end
@@ -79,9 +88,8 @@ module Puppet::Util::Checksums
     Digest::MD5.hexdigest(content)
   end
 
-  # Calculate a checksum of the first 500 chars of the content using Digest::MD5.
-  def md5lite(content)
-    md5(content[0..511])
+  def md5?(string)
+    string =~ /^\h{32}$/
   end
 
   # Calculate a checksum of a file's content using Digest::MD5.
@@ -90,11 +98,6 @@ module Puppet::Util::Checksums
     checksum_file(digest, filename,  lite)
   end
 
-  # Calculate a checksum of the first 500 chars of a file's content using Digest::MD5.
-  def md5lite_file(filename)
-    md5_file(filename, true)
-  end
-
   def md5_stream(lite = false, &block)
     digest = Digest::MD5.new
     checksum_stream(digest, block, lite)
@@ -104,6 +107,20 @@ module Puppet::Util::Checksums
     32
   end
 
+  # Calculate a checksum of the first 500 chars of the content using Digest::MD5.
+  def md5lite(content)
+    md5(content[0..511])
+  end
+
+  def md5lite?(string)
+    md5?(string)
+  end
+
+  # Calculate a checksum of the first 500 chars of a file's content using Digest::MD5.
+  def md5lite_file(filename)
+    md5_file(filename, true)
+  end
+
   def md5lite_stream(&block)
     md5_stream(true, &block)
   end
@@ -112,6 +129,17 @@ module Puppet::Util::Checksums
     md5_hex_length
   end
 
+  def mtime(content)
+    ""
+  end
+
+  def mtime?(string)
+    return true if string.is_a? Time
+    !!DateTime.parse(string)
+  rescue
+    false
+  end
+
   # Return the :mtime timestamp of a file.
   def mtime_file(filename)
     Puppet::FileSystem.stat(filename).send(:mtime)
@@ -125,18 +153,13 @@ module Puppet::Util::Checksums
     nil
   end
 
-  def mtime(content)
-    ""
-  end
-
   # Calculate a checksum using Digest::SHA1.
   def sha1(content)
     Digest::SHA1.hexdigest(content)
   end
 
-  # Calculate a checksum of the first 500 chars of the content using Digest::SHA1.
-  def sha1lite(content)
-    sha1(content[0..511])
+  def sha1?(string)
+    string =~ /^\h{40}$/
   end
 
   # Calculate a checksum of a file's content using Digest::SHA1.
@@ -145,11 +168,6 @@ module Puppet::Util::Checksums
     checksum_file(digest, filename, lite)
   end
 
-  # Calculate a checksum of the first 500 chars of a file's content using Digest::SHA1.
-  def sha1lite_file(filename)
-    sha1_file(filename, true)
-  end
-
   def sha1_stream(lite = false, &block)
     digest = Digest::SHA1.new
     checksum_stream(digest, block, lite)
@@ -159,6 +177,20 @@ module Puppet::Util::Checksums
     40
   end
 
+  # Calculate a checksum of the first 500 chars of the content using Digest::SHA1.
+  def sha1lite(content)
+    sha1(content[0..511])
+  end
+
+  def sha1lite?(string)
+    sha1?(string)
+  end
+
+  # Calculate a checksum of the first 500 chars of a file's content using Digest::SHA1.
+  def sha1lite_file(filename)
+    sha1_file(filename, true)
+  end
+
   def sha1lite_stream(&block)
     sha1_stream(true, &block)
   end
@@ -167,6 +199,17 @@ module Puppet::Util::Checksums
     sha1_hex_length
   end
 
+  def ctime(content)
+    ""
+  end
+
+  def ctime?(string)
+    return true if string.is_a? Time
+    !!DateTime.parse(string)
+  rescue
+    false
+  end
+
   # Return the :ctime of a file.
   def ctime_file(filename)
     Puppet::FileSystem.stat(filename).send(:ctime)
@@ -176,10 +219,14 @@ module Puppet::Util::Checksums
     mtime_stream(&block)
   end
 
-  def ctime(content)
+  def none(content)
     ""
   end
 
+  def none?(string)
+    string.empty?
+  end
+
   # Return a "no checksum"
   def none_file(filename)
     ""
@@ -191,10 +238,6 @@ module Puppet::Util::Checksums
     ""
   end
 
-  def none(content)
-    ""
-  end
-
   class DigestLite
     def initialize(digest, lite = false)
       @digest = digest

data/lib/puppet/util/classgen.rb

@@ -153,7 +153,7 @@ module Puppet::Util::ClassGen
   # @api private
   #
   def handleclassconst(klass, name, options)
-    const = genconst_string(name, options)
+   const = genconst_string(name, options)
 
     if is_constant_defined?(const)
       if options[:overwrite]

data/lib/puppet/util/command_line/trollop.rb

@@ -141,7 +141,7 @@ class Parser
   ## There's one ambiguous case to be aware of: when +:multi+: is true and a
   ## +:default+ is set to an array (of something), it's ambiguous whether this
   ## is a multi-value argument as well as a multi-occurrence argument.
-  ## In thise case, Trollop assumes that it's not a multi-value argument.
+  ## In this case, Trollop assumes that it's not a multi-value argument.
   ## If you want a multi-value, multi-occurrence argument with a default
   ## value, you must specify +:type+ as well.
 

data/lib/puppet/util/command_line.rb

@@ -81,7 +81,11 @@ module Puppet
 
       def find_subcommand
         if subcommand_name.nil?
-          NilSubcommand.new(self)
+          if args.include?("--help") || args.include?("-h")
+            ApplicationSubcommand.new("help", CommandLine.new("puppet", ["help"]))
+          else
+            NilSubcommand.new(self)
+          end
         elsif Puppet::Application.available_application_names.include?(subcommand_name)
           ApplicationSubcommand.new(subcommand_name, self)
         elsif path_to_subcommand = external_subcommand
@@ -102,7 +106,7 @@ module Puppet
           # For most applications, we want to be able to load code from the modulepath,
           # such as apply, describe, resource, and faces.
           # For agent, we only want to load pluginsync'ed code from libdir.
-          # For master, we shouldn't ever be loading per-enviroment code into the master's
+          # For master, we shouldn't ever be loading per-environment code into the master's
           # ruby process, but that requires fixing (#17210, #12173, #8750). So for now
           # we try to restrict to only code that can be autoloaded from the node's
           # environment.
@@ -117,6 +121,10 @@ module Puppet
               configured_environment.each_plugin_directory do |dir|
                 $LOAD_PATH << dir unless $LOAD_PATH.include?(dir)
               end
+
+              # Puppet requires Facter, which initializes its lookup paths. Reset Facter to
+              # pickup the new $LOAD_PATH.
+              Facter.reset
             end
           end
 

data/lib/puppet/util/execution.rb

@@ -1,3 +1,4 @@
+require 'timeout'
 require 'puppet/file_system/uniquefile'
 
 module Puppet
@@ -10,7 +11,7 @@ module Puppet
   end
 end
 
-# This module defines methods for execution of system commands. It is intented for inclusion
+# This module defines methods for execution of system commands. It is intended for inclusion
 # in classes that needs to execute system commands.
 # @api public
 module Puppet::Util::Execution
@@ -47,7 +48,7 @@ module Puppet::Util::Execution
   # @yield [pipe] to a block executing a subprocess
   # @yieldparam pipe [IO] the opened pipe
   # @yieldreturn [String] the output to return
-  # @raise [Puppet::ExecutionFailure] if the executed chiled process did not
+  # @raise [Puppet::ExecutionFailure] if the executed child process did not
   #   exit with status == 0 and `failonfail` is `true`.
   # @return [String] a string with the output from the subprocess executed by
   #   the given block
@@ -119,10 +120,10 @@ module Puppet::Util::Execution
   #   user id matches the effective user id of the current process.
   # @option options [Integer, String] :gid (nil) the group id of the group that the process should be run as. Will be ignored if the
   #   group id matches the effective group id of the current process.
-  # @option options [Boolean] :combine sets whether or not to combine stdout/stderr in the output
+  # @option options [Boolean] :combine sets whether or not to combine stdout/stderr in the output, if false stderr output is discarded
   # @option options [String] :stdinfile (nil) sets a file that can be used for stdin. Passing a string for stdin is not currently
   #   supported.
-  # @option options [Boolean] :squelch (true) if true, ignore stdout / stderr completely.
+  # @option options [Boolean] :squelch (false) if true, ignore stdout / stderr completely.
   # @option options [Boolean] :override_locale (true) by default (and if this option is set to true), we will temporarily override
   #   the user/system locale to "C" (via environment variables LANG and LC_*) while we are executing the command.
   #   This ensures that the output of the command will be formatted consistently, making it predictable for parsing.
@@ -183,17 +184,32 @@ module Puppet::Util::Execution
     null_file = Puppet.features.microsoft_windows? ? 'NUL' : '/dev/null'
 
     begin
-      stdin = File.open(options[:stdinfile] || null_file, 'r')
-      stdout = options[:squelch] ? File.open(null_file, 'w') : Puppet::FileSystem::Uniquefile.new('puppet')
-      stderr = options[:combine] ? stdout : File.open(null_file, 'w')
+      stdin = Puppet::FileSystem.open(options[:stdinfile] || null_file, nil, 'r')
+      stdout = options[:squelch] ? Puppet::FileSystem.open(null_file, nil, 'w') : Puppet::FileSystem::Uniquefile.new('puppet')
+      stderr = options[:combine] ? stdout : Puppet::FileSystem.open(null_file, nil, 'w')
 
       exec_args = [command, options, stdin, stdout, stderr]
 
       if execution_stub = Puppet::Util::ExecutionStub.current_value
         return execution_stub.call(*exec_args)
       elsif Puppet.features.posix?
-        child_pid = execute_posix(*exec_args)
-        exit_status = Process.waitpid2(child_pid).last.exitstatus
+        child_pid = nil
+        begin
+          child_pid = execute_posix(*exec_args)
+          exit_status = Process.waitpid2(child_pid).last.exitstatus
+          child_pid = nil
+        rescue Timeout::Error => e
+          # NOTE: For Ruby 2.1+, an explicit Timeout::Error class has to be
+          # passed to Timeout.timeout in order for there to be something for
+          # this block to rescue.
+          unless child_pid.nil?
+            Process.kill(:TERM, child_pid)
+            # Spawn a thread to reap the process if it dies.
+            Thread.new { Process.waitpid(child_pid) }
+          end
+
+          raise e
+        end
       elsif Puppet.features.microsoft_windows?
         process_info = execute_windows(*exec_args)
         begin
@@ -301,7 +317,7 @@ module Puppet::Util::Execution
     end.join(" ") if command.is_a?(Array)
 
     options[:custom_environment] ||= {}
-    Puppet::Util.withenv(options[:custom_environment]) do
+    Puppet::Util.withenv(options[:custom_environment], :windows) do
       Puppet::Util::Windows::Process.execute(command, options, stdin, stdout, stderr)
     end
   end

data/lib/puppet/util/feature.rb

@@ -32,7 +32,7 @@ class Puppet::Util::Feature
       #    configured to always cache
       if block_given?     ||
           @results[name]  ||
-          (@results.has_key?(name) and Puppet[:always_cache_features])
+          (@results.has_key?(name) && (Puppet[:always_cache_features] || !Puppet[:always_retry_plugins]))
         @results[name]
       else
         @results[name] = test(name, options)

data/lib/puppet/util/filetype.rb

@@ -105,7 +105,9 @@ class Puppet::Util::FileType
     # Read the file.
     def read
       if Puppet::FileSystem.exist?(@path)
-        File.read(@path)
+        # this code path is used by many callers so the original default is
+        # being explicitly preserved
+        Puppet::FileSystem.read(@path, :encoding => Encoding.default_external)
       else
         return nil
       end
@@ -118,7 +120,8 @@ class Puppet::Util::FileType
 
     # Overwrite the file.
     def write(text)
-      tf = Tempfile.new("puppet")
+      # this file is managed by the OS and should be using system encoding
+      tf = Tempfile.new("puppet", :encoding => Encoding.default_external)
       tf.print text; tf.flush
       File.chmod(@default_mode, tf.path) if @default_mode
       FileUtils.cp(tf.path, @path)
@@ -197,7 +200,8 @@ class Puppet::Util::FileType
     # Overwrite a specific @path's cron tab; must be passed the @path name
     # and the text with which to create the cron tab.
     def write(text)
-      IO.popen("#{cmdbase()} -", "w") { |p|
+      # this file is managed by the OS and should be using system encoding
+      IO.popen("#{cmdbase()} -", "w", :encoding => Encoding.default_external) { |p|
         p.print text
       }
     end
@@ -242,7 +246,8 @@ class Puppet::Util::FileType
     # Overwrite a specific @path's cron tab; must be passed the @path name
     # and the text with which to create the cron tab.
     def write(text)
-      output_file = Tempfile.new("puppet_suntab")
+      # this file is managed by the OS and should be using system encoding
+      output_file = Tempfile.new("puppet_suntab", :encoding => Encoding.default_external)
       begin
         output_file.print text
         output_file.close
@@ -284,7 +289,8 @@ class Puppet::Util::FileType
     # Overwrite a specific @path's cron tab; must be passed the @path name
     # and the text with which to create the cron tab.
     def write(text)
-      output_file = Tempfile.new("puppet_aixtab")
+      # this file is managed by the OS and should be using system encoding
+      output_file = Tempfile.new("puppet_aixtab", :encoding => Encoding.default_external)
 
       begin
         output_file.print text

data/lib/puppet/util/http_proxy.rb

@@ -1,4 +1,16 @@
+require 'uri'
+require 'openssl'
+
 module Puppet::Util::HttpProxy
+  def self.proxy(uri)
+    if self.no_proxy?(uri)
+      proxy_class = Net::HTTP::Proxy(nil)
+    else
+      proxy_class = Net::HTTP::Proxy(self.http_proxy_host, self.http_proxy_port, self.http_proxy_user, self.http_proxy_password)
+    end
+
+    return proxy_class.new(uri.host, uri.port)
+  end
 
   def self.http_proxy_env
     # Returns a URI object if proxy is set, or nil
@@ -11,6 +23,57 @@ module Puppet::Util::HttpProxy
     return nil
   end
 
+  # The documentation around the format of the no_proxy variable seems
+  # inconsistent.  Some suggests the use of the * as a way of matching any
+  # hosts under a domain, e.g.:
+  #   *.example.com
+  # Other documentation suggests that just a leading '.' indicates a domain
+  # level exclusion, e.g.:
+  #   .example.com
+  # We'll accommodate both here.
+  def self.no_proxy?(dest)
+    unless no_proxy_env = ENV["no_proxy"] || ENV["NO_PROXY"]
+      return false
+    end
+
+    unless dest.is_a? URI
+      begin
+        dest = URI.parse(dest)
+      rescue URI::InvalidURIError
+        return false
+      end
+    end
+
+    no_proxy_env.split(/\s*,\s*/).each do |d|
+      host, port = d.split(':')
+      host = Regexp.escape(host).gsub('\*', '.*')
+
+      #If the host of this no_proxy value starts with '.', this entry is
+      #a domain level entry. Don't pin the regex to the beginning of the entry.
+      #If it does not start with a '.' then it is a host specific entry and
+      #should be matched to the destination starting at the beginning.
+      unless host =~ /^\\\./
+        host = "^#{host}"
+      end
+
+      #If this no_proxy entry specifies a port, we want to match it against
+      #the destination port.  Otherwise just match hosts.
+      if port
+        no_proxy_regex  = %r(#{host}:#{port}$)
+        dest_string     = "#{dest.host}:#{dest.port}"
+      else
+        no_proxy_regex  = %r(#{host}$)
+        dest_string     = "#{dest.host}"
+      end
+
+      if no_proxy_regex.match(dest_string)
+        return true
+      end
+    end
+
+    return false
+  end
+
   def self.http_proxy_host
     env = self.http_proxy_env
 
@@ -62,4 +125,73 @@ module Puppet::Util::HttpProxy
 
     return Puppet.settings[:http_proxy_password]
   end
+
+  # Return a Net::HTTP::Proxy object.
+  #
+  # This method optionally configures SSL correctly if the URI scheme is
+  # 'https', including setting up the root certificate store so remote server
+  # SSL certificates can be validated.
+  #
+  # @param [URI] uri The URI that is to be accessed.
+  # @return [Net::HTTP::Proxy] object constructed tailored for the passed URI
+  def self.get_http_object(uri)
+    proxy = proxy(uri)
+
+    if uri.scheme == 'https'
+      cert_store = OpenSSL::X509::Store.new
+      cert_store.set_default_paths
+
+      proxy.use_ssl = true
+      proxy.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      proxy.cert_store = cert_store
+    end
+
+    if Puppet[:http_debug]
+      proxy.set_debug_output($stderr)
+    end
+
+    proxy.open_timeout = Puppet[:http_connect_timeout]
+    proxy.read_timeout = Puppet[:http_read_timeout]
+
+    proxy
+  end
+
+  # Retrieve a document through HTTP(s), following redirects if necessary.
+  # 
+  # Based on the the client implementation in the HTTP pool.
+  #
+  # @see Puppet::Network::HTTP::Connection#request_with_redirects
+  #
+  # @param [URI] uri The address of the resource to retrieve.
+  # @param [symbol] method The name of the Net::HTTP method to use, typically :get, :head, :post etc.
+  # @param [FixNum] redirect_limit The number of redirections that can be followed.
+  # @return [Net::HTTPResponse] a response object
+  def self.request_with_redirects(uri, method, redirect_limit = 10, &block)
+    current_uri = uri
+    response = nil
+
+    0.upto(redirect_limit) do |redirection|
+      proxy = get_http_object(current_uri)
+      response = proxy.send(:head, current_uri.path)
+
+      if [301, 302, 307].include?(response.code.to_i)
+        # handle the redirection
+        current_uri = URI.parse(response['location'])
+        next
+      end
+
+      if block_given?
+        headers = {'Accept' => 'binary', 'accept-encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3'}
+        response = proxy.send("request_#{method}".to_sym, current_uri.path, headers, &block)
+      else
+        response = proxy.send(method, current_uri.path)
+      end
+
+      Puppet.debug("HTTP #{method.to_s.upcase} request to #{current_uri} returned #{response.code} #{response.message}")
+
+      return response
+    end
+
+    raise RedirectionLimitExceededException, "Too many HTTP redirections for #{uri}"
+  end
 end

data/lib/puppet/util/inifile.rb

@@ -56,7 +56,7 @@ module Puppet::Util::IniConfig
     end
 
     # Set the entry 'key=value'. If no entry with the
-    # given key exists, one is appended to teh end of the section
+    # given key exists, one is appended to the end of the section
     def []=(key, value)
       entry = find_entry(key)
       @dirty = true

data/lib/puppet/util/instance_loader.rb

@@ -60,7 +60,7 @@ module Puppet::Util::InstanceLoader
     @autoloaders[type.intern]
   end
 
-  # Retrieve an alread-loaded instance, or attempt to load our instance.
+  # Retrieve an already-loaded instance, or attempt to load our instance.
   def loaded_instance(type, name)
     name = name.intern
     return nil unless instances = instance_hash(type)

data/lib/puppet/util/log/destinations.rb

@@ -65,6 +65,8 @@ Puppet::Util::Log.newdesttype :file do
 
   def initialize(path)
     @name = path
+    @json = path.end_with?('.json') ? 1 : 0
+
     # first make sure the directory exists
     # We can't just use 'Config.use' here, because they've
     # specified a "special" destination.
@@ -74,7 +76,21 @@ Puppet::Util::Log.newdesttype :file do
     end
 
     # create the log file, if it doesn't already exist
-    file = File.open(path, File::WRONLY|File::CREAT|File::APPEND)
+    need_array_start = false
+    if @json == 1
+      need_array_start = true
+      if File.exists?(path)
+        sz = File.size(path)
+        need_array_start = sz == 0
+
+        # Assume that entries have been written and that a comma
+        # is needed before next entry
+        @json = 2 if sz > 2
+      end
+    end
+
+    file = File.open(path,  File::WRONLY|File::CREAT|File::APPEND)
+    file.puts('[') if need_array_start
 
     # Give ownership to the user and group puppet will run as
     if Puppet.features.root? && !Puppet::Util::Platform.windows?
@@ -91,7 +107,12 @@ Puppet::Util::Log.newdesttype :file do
   end
 
   def handle(msg)
-    @file.puts("#{msg.time} #{msg.source} (#{msg.level}): #{msg}")
+    if @json > 0
+      @json > 1 ? @file.puts(',') : @json = 2
+      JSON.dump(msg.to_structured_hash, @file)
+    else
+      @file.puts("#{msg.time} #{msg.source} (#{msg.level}): #{msg}")
+    end
 
     @file.flush if @autoflush
   end