puppet 4.1.0 → 4.10.0

data/lib/puppet/util/autoload.rb

@@ -2,6 +2,15 @@ require 'pathname'
 require 'puppet/util/rubygems'
 require 'puppet/util/warnings'
 require 'puppet/util/methodhelper'
+require 'puppet/pops/adaptable'
+
+# An adapter that ties the module_directories cache to the environment where the modules are parsed. This
+# adapter ensures that the life-cycle of this cache doesn't exceed  the life-cycle of the environment.
+#
+# @api private
+class Puppet::Util::ModuleDirectoriesAdapter < Puppet::Pops::Adaptable::Adapter
+  attr_accessor :directories
+end
 
 # Autoload paths, either based on names or all at once.
 class Puppet::Util::Autoload
@@ -99,12 +108,6 @@ class Puppet::Util::Autoload
     end
 
     def module_directories(env)
-      # We're using a per-thread cache of module directories so that we don't
-      # scan the filesystem each time we try to load something. This is reset
-      # at the beginning of compilation and at the end of an agent run.
-      $env_module_directories ||= {}
-
-
       # This is a little bit of a hack.  Basically, the autoloader is being
       # called indirectly during application bootstrapping when we do things
       # such as check "features".  However, during bootstrapping, we haven't
@@ -129,11 +132,13 @@ class Puppet::Util::Autoload
 
         if env
           # if the app defaults have been initialized then it should be safe to access the module path setting.
-          $env_module_directories[env] ||= env.modulepath.collect do |dir|
-            Dir.entries(dir).reject { |f| f =~ /^\./ }.collect { |f| File.join(dir, f, "lib") }
-          end.flatten.find_all do |d|
-            FileTest.directory?(d)
-          end
+          Puppet::Util::ModuleDirectoriesAdapter.adapt(env) do |a|
+            a.directories ||= env.modulepath.collect do |dir|
+              Dir.entries(dir).reject { |f| f =~ /^\./ }.collect { |f| File.join(dir, f, "lib") }
+            end.flatten.find_all do |d|
+              FileTest.directory?(d)
+            end
+          end.directories
         else
           []
         end

data/lib/puppet/util/character_encoding.rb

@@ -0,0 +1,95 @@
+# A module to centralize heuristics/practices for managing character encoding in Puppet
+
+module Puppet::Util::CharacterEncoding
+  class << self
+    # Warning! This is a destructive method - the string supplied is modified!
+    # @api public
+    # @param [String] string a string to transcode / force_encode to utf-8
+    # @return [String] string if already utf-8, OR
+    #   the same string with external encoding set to utf-8 if bytes are valid utf-8 OR
+    #   the same string transcoded to utf-8 OR
+    #   nil upon a failure to legitimately set external encoding or transcode string
+    def convert_to_utf_8!(string)
+      currently_valid = string.valid_encoding?
+
+      begin
+        if string.encoding == Encoding::UTF_8
+          if currently_valid
+            return string
+          else
+            # If a string is currently believed to be UTF-8, but is also not
+            # valid_encoding?, we have no recourse but to fail because we have no
+            # idea what encoding this string originally came from where it *was*
+            # valid - all we know is it's not currently valid UTF-8.
+            raise EncodingError
+          end
+        elsif valid_utf_8_bytes?(string)
+          # Before we try to transcode the string, check if it is valid UTF-8 as
+          # currently constitued (in its non-UTF-8 encoding), and if it is, limit
+          # ourselves to setting the external encoding of the string to UTF-8
+          # rather than actually transcoding it. We do this to handle
+          # a couple scenarios:
+
+          # The first scenario is that the string was originally valid UTF-8 but
+          # the current puppet run is not in a UTF-8 environment. In this case,
+          # the string will likely have invalid byte sequences (i.e.,
+          # string.valid_encoding? == false), and attempting to transcode will
+          # fail with Encoding::InvalidByteSequenceError, referencing the
+          # invalid byte sequence in the original, pre-transcode, string. We
+          # might have gotten here, for example, if puppet is run first in a
+          # user context with UTF-8 encoding (setting the "is" value to UTF-8)
+          # and then later run via cron without UTF-8 specified, resulting in in
+          # EN_US (ISO-8859-1) on many systems. In this scenario we're
+          # effectively best-guessing this string originated as UTF-8 and only
+          # set external encoding to UTF-8 - transcoding would have failed
+          # anyway.
+
+          # The second scenario (more rare, I expect) is that this string does
+          # NOT have invalid byte sequences (string.valid_encoding? == true),
+          # but is *ALSO valid unicode*.
+          # Our example case is "\u16A0" - "RUNIC LETTER FEHU FEOH FE"
+          # http://www.fileformat.info/info/unicode/char/16A0/index.htm
+          # 0xE1 0x9A 0xA0 / 225 154 160
+          # These bytes are valid in ISO-8859-1 but the character they represent
+          # transcodes cleanly in ruby to *different* characters in UTF-8.
+          # That's not what we want if the user intended the original string as
+          # UTF-8. We can only guess, so if the string is valid UTF-8 as
+          # currently constituted, we default to assuming the string originated
+          # in UTF-8 and do not transcode it - we only set external encoding.
+          return string.force_encoding(Encoding::UTF_8)
+        elsif currently_valid
+          # If the string is not currently valid UTF-8 but it can be transcoded
+          # (it is valid in its current encoding), we can guess this string was
+          # not originally unicode. Transcode it to UTF-8. For strings with
+          # original encodings like SHIFT_JIS, this should be the final result.
+          return string.encode!(Encoding::UTF_8)
+        else
+          # If the string is neither valid UTF-8 as-is nor valid in its current
+          # encoding, fail. It requires user remediation.
+          raise EncodingError
+        end
+      rescue EncodingError => detail
+        # Catch both our own self-determined failure to transcode as well as any
+        # error on ruby's part, ie Encoding::UndefinedConversionError on a
+        # failure to encode!.
+        Puppet.debug(_("%{error}: %{value} is not valid UTF-8 and cannot be transcoded by Puppet.") %
+          { error: detail.inspect, value: string.dump })
+        return nil
+      end
+    end
+
+    private
+
+    # Do our best to determine if a string is valid UTF-8 via String#valid_encoding? without permanently
+    # modifying or duplicating the string due to performance concerns
+    # @api private
+    # @param [String] string a string to test
+    # @return [Boolean] whether we think the string is UTF-8 or not
+    def valid_utf_8_bytes?(string)
+      original_encoding = string.encoding
+      valid = string.force_encoding(Encoding::UTF_8).valid_encoding?
+      string.force_encoding(original_encoding)
+      valid
+    end
+  end
+end

data/lib/puppet/util/checksums.rb

@@ -1,5 +1,6 @@
 require 'digest/md5'
 require 'digest/sha1'
+require 'time'
 
 # A stand-alone module for calculating checksums
 # in a generic way.
@@ -41,8 +42,8 @@ module Puppet::Util::Checksums
     Digest::SHA256.hexdigest(content)
   end
 
-  def sha256lite(content)
-    sha256(content[0..511])
+  def sha256?(string)
+    string =~ /^\h{64}$/
   end
 
   def sha256_file(filename, lite = false)
@@ -52,10 +53,6 @@ module Puppet::Util::Checksums
     checksum_file(digest, filename,  lite)
   end
 
-  def sha256lite_file(filename)
-    sha256_file(filename, true)
-  end
-
   def sha256_stream(lite = false, &block)
     require 'digest/sha2'
     digest = Digest::SHA256.new
@@ -66,6 +63,18 @@ module Puppet::Util::Checksums
     64
   end
 
+  def sha256lite(content)
+    sha256(content[0..511])
+  end
+
+  def sha256lite?(string)
+    sha256?(string)
+  end
+
+  def sha256lite_file(filename)
+    sha256_file(filename, true)
+  end
+
   def sha256lite_stream(&block)
     sha256_stream(true, &block)
   end
@@ -79,9 +88,8 @@ module Puppet::Util::Checksums
     Digest::MD5.hexdigest(content)
   end
 
-  # Calculate a checksum of the first 500 chars of the content using Digest::MD5.
-  def md5lite(content)
-    md5(content[0..511])
+  def md5?(string)
+    string =~ /^\h{32}$/
   end
 
   # Calculate a checksum of a file's content using Digest::MD5.
@@ -90,11 +98,6 @@ module Puppet::Util::Checksums
     checksum_file(digest, filename,  lite)
   end
 
-  # Calculate a checksum of the first 500 chars of a file's content using Digest::MD5.
-  def md5lite_file(filename)
-    md5_file(filename, true)
-  end
-
   def md5_stream(lite = false, &block)
     digest = Digest::MD5.new
     checksum_stream(digest, block, lite)
@@ -104,6 +107,20 @@ module Puppet::Util::Checksums
     32
   end
 
+  # Calculate a checksum of the first 500 chars of the content using Digest::MD5.
+  def md5lite(content)
+    md5(content[0..511])
+  end
+
+  def md5lite?(string)
+    md5?(string)
+  end
+
+  # Calculate a checksum of the first 500 chars of a file's content using Digest::MD5.
+  def md5lite_file(filename)
+    md5_file(filename, true)
+  end
+
   def md5lite_stream(&block)
     md5_stream(true, &block)
   end
@@ -112,6 +129,17 @@ module Puppet::Util::Checksums
     md5_hex_length
   end
 
+  def mtime(content)
+    ""
+  end
+
+  def mtime?(string)
+    return true if string.is_a? Time
+    !!DateTime.parse(string)
+  rescue
+    false
+  end
+
   # Return the :mtime timestamp of a file.
   def mtime_file(filename)
     Puppet::FileSystem.stat(filename).send(:mtime)
@@ -125,18 +153,13 @@ module Puppet::Util::Checksums
     nil
   end
 
-  def mtime(content)
-    ""
-  end
-
   # Calculate a checksum using Digest::SHA1.
   def sha1(content)
     Digest::SHA1.hexdigest(content)
   end
 
-  # Calculate a checksum of the first 500 chars of the content using Digest::SHA1.
-  def sha1lite(content)
-    sha1(content[0..511])
+  def sha1?(string)
+    string =~ /^\h{40}$/
   end
 
   # Calculate a checksum of a file's content using Digest::SHA1.
@@ -145,11 +168,6 @@ module Puppet::Util::Checksums
     checksum_file(digest, filename, lite)
   end
 
-  # Calculate a checksum of the first 500 chars of a file's content using Digest::SHA1.
-  def sha1lite_file(filename)
-    sha1_file(filename, true)
-  end
-
   def sha1_stream(lite = false, &block)
     digest = Digest::SHA1.new
     checksum_stream(digest, block, lite)
@@ -159,6 +177,20 @@ module Puppet::Util::Checksums
     40
   end
 
+  # Calculate a checksum of the first 500 chars of the content using Digest::SHA1.
+  def sha1lite(content)
+    sha1(content[0..511])
+  end
+
+  def sha1lite?(string)
+    sha1?(string)
+  end
+
+  # Calculate a checksum of the first 500 chars of a file's content using Digest::SHA1.
+  def sha1lite_file(filename)
+    sha1_file(filename, true)
+  end
+
   def sha1lite_stream(&block)
     sha1_stream(true, &block)
   end
@@ -167,6 +199,17 @@ module Puppet::Util::Checksums
     sha1_hex_length
   end
 
+  def ctime(content)
+    ""
+  end
+
+  def ctime?(string)
+    return true if string.is_a? Time
+    !!DateTime.parse(string)
+  rescue
+    false
+  end
+
   # Return the :ctime of a file.
   def ctime_file(filename)
     Puppet::FileSystem.stat(filename).send(:ctime)
@@ -176,10 +219,14 @@ module Puppet::Util::Checksums
     mtime_stream(&block)
   end
 
-  def ctime(content)
+  def none(content)
     ""
   end
 
+  def none?(string)
+    string.empty?
+  end
+
   # Return a "no checksum"
   def none_file(filename)
     ""
@@ -191,10 +238,6 @@ module Puppet::Util::Checksums
     ""
   end
 
-  def none(content)
-    ""
-  end
-
   class DigestLite
     def initialize(digest, lite = false)
       @digest = digest

data/lib/puppet/util/classgen.rb

@@ -153,7 +153,7 @@ module Puppet::Util::ClassGen
   # @api private
   #
   def handleclassconst(klass, name, options)
-    const = genconst_string(name, options)
+   const = genconst_string(name, options)
 
     if is_constant_defined?(const)
       if options[:overwrite]

data/lib/puppet/util/command_line/trollop.rb

@@ -141,7 +141,7 @@ class Parser
   ## There's one ambiguous case to be aware of: when +:multi+: is true and a
   ## +:default+ is set to an array (of something), it's ambiguous whether this
   ## is a multi-value argument as well as a multi-occurrence argument.
-  ## In thise case, Trollop assumes that it's not a multi-value argument.
+  ## In this case, Trollop assumes that it's not a multi-value argument.
   ## If you want a multi-value, multi-occurrence argument with a default
   ## value, you must specify +:type+ as well.
 

data/lib/puppet/util/command_line.rb

@@ -81,7 +81,11 @@ module Puppet
 
       def find_subcommand
         if subcommand_name.nil?
-          NilSubcommand.new(self)
+          if args.include?("--help") || args.include?("-h")
+            ApplicationSubcommand.new("help", CommandLine.new("puppet", ["help"]))
+          else
+            NilSubcommand.new(self)
+          end
         elsif Puppet::Application.available_application_names.include?(subcommand_name)
           ApplicationSubcommand.new(subcommand_name, self)
         elsif path_to_subcommand = external_subcommand
@@ -102,7 +106,7 @@ module Puppet
           # For most applications, we want to be able to load code from the modulepath,
           # such as apply, describe, resource, and faces.
           # For agent, we only want to load pluginsync'ed code from libdir.
-          # For master, we shouldn't ever be loading per-enviroment code into the master's
+          # For master, we shouldn't ever be loading per-environment code into the master's
           # ruby process, but that requires fixing (#17210, #12173, #8750). So for now
           # we try to restrict to only code that can be autoloaded from the node's
           # environment.
@@ -117,6 +121,10 @@ module Puppet
               configured_environment.each_plugin_directory do |dir|
                 $LOAD_PATH << dir unless $LOAD_PATH.include?(dir)
               end
+
+              # Puppet requires Facter, which initializes its lookup paths. Reset Facter to
+              # pickup the new $LOAD_PATH.
+              Facter.reset
             end
           end
 

data/lib/puppet/util/execution.rb

@@ -1,3 +1,4 @@
+require 'timeout'
 require 'puppet/file_system/uniquefile'
 
 module Puppet
@@ -10,7 +11,7 @@ module Puppet
   end
 end
 
-# This module defines methods for execution of system commands. It is intented for inclusion
+# This module defines methods for execution of system commands. It is intended for inclusion
 # in classes that needs to execute system commands.
 # @api public
 module Puppet::Util::Execution
@@ -47,7 +48,7 @@ module Puppet::Util::Execution
   # @yield [pipe] to a block executing a subprocess
   # @yieldparam pipe [IO] the opened pipe
   # @yieldreturn [String] the output to return
-  # @raise [Puppet::ExecutionFailure] if the executed chiled process did not
+  # @raise [Puppet::ExecutionFailure] if the executed child process did not
   #   exit with status == 0 and `failonfail` is `true`.
   # @return [String] a string with the output from the subprocess executed by
   #   the given block
@@ -119,10 +120,10 @@ module Puppet::Util::Execution
   #   user id matches the effective user id of the current process.
   # @option options [Integer, String] :gid (nil) the group id of the group that the process should be run as. Will be ignored if the
   #   group id matches the effective group id of the current process.
-  # @option options [Boolean] :combine sets whether or not to combine stdout/stderr in the output
+  # @option options [Boolean] :combine sets whether or not to combine stdout/stderr in the output, if false stderr output is discarded
   # @option options [String] :stdinfile (nil) sets a file that can be used for stdin. Passing a string for stdin is not currently
   #   supported.
-  # @option options [Boolean] :squelch (true) if true, ignore stdout / stderr completely.
+  # @option options [Boolean] :squelch (false) if true, ignore stdout / stderr completely.
   # @option options [Boolean] :override_locale (true) by default (and if this option is set to true), we will temporarily override
   #   the user/system locale to "C" (via environment variables LANG and LC_*) while we are executing the command.
   #   This ensures that the output of the command will be formatted consistently, making it predictable for parsing.
@@ -152,15 +153,18 @@ module Puppet::Util::Execution
         :squelch => false,
         :override_locale => true,
         :custom_environment => {},
+        :sensitive => false,
     }
 
     options = default_options.merge(options)
 
-    if command.is_a?(Array)
+    if options[:sensitive]
+      command_str = '[redacted]'
+    elsif command.is_a?(Array)
       command = command.flatten.map(&:to_s)
-      str = command.join(" ")
+      command_str = command.join(" ")
     elsif command.is_a?(String)
-      str = command
+      command_str = command
     end
 
     user_log_s = ''
@@ -175,25 +179,40 @@ module Puppet::Util::Execution
     end
 
     if respond_to? :debug
-      debug "Executing#{user_log_s}: '#{str}'"
+      debug "Executing#{user_log_s}: '#{command_str}'"
     else
-      Puppet.debug "Executing#{user_log_s}: '#{str}'"
+      Puppet.debug "Executing#{user_log_s}: '#{command_str}'"
     end
 
     null_file = Puppet.features.microsoft_windows? ? 'NUL' : '/dev/null'
 
     begin
-      stdin = File.open(options[:stdinfile] || null_file, 'r')
-      stdout = options[:squelch] ? File.open(null_file, 'w') : Puppet::FileSystem::Uniquefile.new('puppet')
-      stderr = options[:combine] ? stdout : File.open(null_file, 'w')
+      stdin = Puppet::FileSystem.open(options[:stdinfile] || null_file, nil, 'r')
+      stdout = options[:squelch] ? Puppet::FileSystem.open(null_file, nil, 'w') : Puppet::FileSystem::Uniquefile.new('puppet')
+      stderr = options[:combine] ? stdout : Puppet::FileSystem.open(null_file, nil, 'w')
 
       exec_args = [command, options, stdin, stdout, stderr]
 
       if execution_stub = Puppet::Util::ExecutionStub.current_value
         return execution_stub.call(*exec_args)
       elsif Puppet.features.posix?
-        child_pid = execute_posix(*exec_args)
-        exit_status = Process.waitpid2(child_pid).last.exitstatus
+        child_pid = nil
+        begin
+          child_pid = execute_posix(*exec_args)
+          exit_status = Process.waitpid2(child_pid).last.exitstatus
+          child_pid = nil
+        rescue Timeout::Error => e
+          # NOTE: For Ruby 2.1+, an explicit Timeout::Error class has to be
+          # passed to Timeout.timeout in order for there to be something for
+          # this block to rescue.
+          unless child_pid.nil?
+            Process.kill(:TERM, child_pid)
+            # Spawn a thread to reap the process if it dies.
+            Thread.new { Process.waitpid(child_pid) }
+          end
+
+          raise e
+        end
       elsif Puppet.features.microsoft_windows?
         process_info = execute_windows(*exec_args)
         begin
@@ -213,7 +232,7 @@ module Puppet::Util::Execution
       end
 
       if options[:failonfail] and exit_status != 0
-        raise Puppet::ExecutionFailure, "Execution of '#{str}' returned #{exit_status}: #{output.strip}"
+        raise Puppet::ExecutionFailure, "Execution of '#{command_str}' returned #{exit_status}: #{output.strip}"
       end
     ensure
       if !options[:squelch] && stdout
@@ -301,7 +320,7 @@ module Puppet::Util::Execution
     end.join(" ") if command.is_a?(Array)
 
     options[:custom_environment] ||= {}
-    Puppet::Util.withenv(options[:custom_environment]) do
+    Puppet::Util.withenv(options[:custom_environment], :windows) do
       Puppet::Util::Windows::Process.execute(command, options, stdin, stdout, stderr)
     end
   end

data/lib/puppet/util/feature.rb

@@ -32,7 +32,7 @@ class Puppet::Util::Feature
       #    configured to always cache
       if block_given?     ||
           @results[name]  ||
-          (@results.has_key?(name) and Puppet[:always_cache_features])
+          (@results.has_key?(name) && (Puppet[:always_cache_features] || !Puppet[:always_retry_plugins]))
         @results[name]
       else
         @results[name] = test(name, options)

data/lib/puppet/util/filetype.rb

@@ -105,7 +105,9 @@ class Puppet::Util::FileType
     # Read the file.
     def read
       if Puppet::FileSystem.exist?(@path)
-        File.read(@path)
+        # this code path is used by many callers so the original default is
+        # being explicitly preserved
+        Puppet::FileSystem.read(@path, :encoding => Encoding.default_external)
       else
         return nil
       end
@@ -118,7 +120,8 @@ class Puppet::Util::FileType
 
     # Overwrite the file.
     def write(text)
-      tf = Tempfile.new("puppet")
+      # this file is managed by the OS and should be using system encoding
+      tf = Tempfile.new("puppet", :encoding => Encoding.default_external)
       tf.print text; tf.flush
       File.chmod(@default_mode, tf.path) if @default_mode
       FileUtils.cp(tf.path, @path)
@@ -197,7 +200,8 @@ class Puppet::Util::FileType
     # Overwrite a specific @path's cron tab; must be passed the @path name
     # and the text with which to create the cron tab.
     def write(text)
-      IO.popen("#{cmdbase()} -", "w") { |p|
+      # this file is managed by the OS and should be using system encoding
+      IO.popen("#{cmdbase()} -", "w", :encoding => Encoding.default_external) { |p|
         p.print text
       }
     end
@@ -242,7 +246,8 @@ class Puppet::Util::FileType
     # Overwrite a specific @path's cron tab; must be passed the @path name
     # and the text with which to create the cron tab.
     def write(text)
-      output_file = Tempfile.new("puppet_suntab")
+      # this file is managed by the OS and should be using system encoding
+      output_file = Tempfile.new("puppet_suntab", :encoding => Encoding.default_external)
       begin
         output_file.print text
         output_file.close
@@ -284,7 +289,8 @@ class Puppet::Util::FileType
     # Overwrite a specific @path's cron tab; must be passed the @path name
     # and the text with which to create the cron tab.
     def write(text)
-      output_file = Tempfile.new("puppet_aixtab")
+      # this file is managed by the OS and should be using system encoding
+      output_file = Tempfile.new("puppet_aixtab", :encoding => Encoding.default_external)
 
       begin
         output_file.print text