puppet 4.9.4 → 4.10.0

data/lib/puppet/provider/service/systemd.rb

@@ -23,7 +23,7 @@ Puppet::Type.type(:service).provide :systemd, :parent => :base do
   defaultfor :osfamily => :redhat, :operatingsystem => :fedora
   defaultfor :osfamily => :suse
   defaultfor :osfamily => :coreos
-  defaultfor :operatingsystem => :debian, :operatingsystemmajrelease => "8"
+  defaultfor :operatingsystem => :debian, :operatingsystemmajrelease => ["8", "stretch/sid", "9", "buster/sid"]
   defaultfor :operatingsystem => :ubuntu, :operatingsystemmajrelease => ["15.04","15.10","16.04","16.10"]
   defaultfor :operatingsystem => :cumuluslinux, :operatingsystemmajrelease => ["3"]
 

data/lib/puppet/provider/user/aix.rb

@@ -216,7 +216,9 @@ Puppet::Type.type(:user).provide :aix, :parent => Puppet::Provider::AixObject do
 
   def open_security_passwd
     # helper method for tests
-    File.open("/etc/security/passwd", 'r')
+    # AIX reference indicates this file is ASCII
+    # https://www.ibm.com/support/knowledgecenter/en/ssw_aix_72/com.ibm.aix.files/passwd_security.htm
+    Puppet::FileSystem.open("/etc/security/passwd", nil, "r:ASCII")
   end
 
   #--------------------------------
@@ -255,7 +257,10 @@ Puppet::Type.type(:user).provide :aix, :parent => Puppet::Provider::AixObject do
     user = @resource[:name]
 
     # Puppet execute does not support strings as input, only files.
-    tmpfile = Tempfile.new('puppet_#{user}_pw')
+    # The password is expected to be in an encrypted format given -e is specified:
+    # https://www.ibm.com/support/knowledgecenter/ssw_aix_71/com.ibm.aix.cmds1/chpasswd.htm
+    # /etc/security/passwd is specified as an ASCII file per the AIX documentation
+    tmpfile = Tempfile.new("puppet_#{user}_pw", :encoding => Encoding::ASCII)
     tmpfile << "#{user}:#{value}\n"
     tmpfile.close()
 

data/lib/puppet/settings.rb

@@ -44,6 +44,8 @@ class Puppet::Settings
   # The acceptable sections of the puppet.conf configuration file.
   ALLOWED_SECTION_NAMES = ['main', 'master', 'agent', 'user'].freeze
 
+  NONE = 'none'.freeze
+
   # This method is intended for puppet internal use only; it is a convenience method that
   # returns reasonable application default settings values for a given run_mode.
   def self.app_defaults_for_run_mode(run_mode)
@@ -384,12 +386,12 @@ class Puppet::Settings
     end
   end
 
-  def_delegator :@config, :each
+  def_delegators :@config, :each, :each_pair, :each_key
 
   # Iterate over each section name.
   def eachsection
     yielded = []
-    @config.each do |name, object|
+    @config.each_value do |object|
       section = object.section
       unless yielded.include? section
         yield section
@@ -549,7 +551,7 @@ class Puppet::Settings
     if @config[:environment]
       env = self.value(:environment).to_sym
     else
-      env = "none"
+      env = NONE
     end
 
     # Call any hooks we should be calling.
@@ -1046,26 +1048,37 @@ Generated on #{Time.now}.
   #
   # @raise [InterpolationError]
   def value(param, environment = nil, bypass_interpolation = false)
-    param = param.to_sym
     environment &&= environment.to_sym
+    value_sym(param.to_sym, environment, bypass_interpolation)
+  end
 
-    setting = @config[param]
-
-    # Short circuit to nil for undefined settings.
-    return nil if setting.nil?
-
+  # Find the correct value using symbols and our search path.
+  #
+  # @param param [Symbol] The value to look up
+  # @param environment [Symbol] The environment to check for the value
+  # @param bypass_interpolation [true, false] Whether to skip interpolation
+  #
+  # @return [Object] The looked up value
+  #
+  # @raise [InterpolationError]
+  def value_sym(param, environment = nil, bypass_interpolation = false)
     # Check the cache first.  It needs to be a per-environment
     # cache so that we don't spread values from one env
     # to another.
-    if @cache[environment||"none"].has_key?(param)
-      return @cache[environment||"none"][param]
-    elsif bypass_interpolation
-      val = values(environment, self.preferred_run_mode).lookup(param)
-    else
-      val = values(environment, self.preferred_run_mode).interpolate(param)
-    end
+    cached_env = @cache[environment || NONE]
+
+    # Avoid two lookups in cache_env unless val is nil. When it is, it's important
+    # to check if the key is included so that further processing (that will result
+    # in nil again) is avoided.
+    val = cached_env[param]
+    return val if !val.nil? || cached_env.include?(param)
+
+    # Short circuit to nil for undefined settings.
+    return nil unless @config.include?(param)
 
-    @cache[environment||"none"][param] = val
+    vals = values(environment, preferred_run_mode)
+    val = bypass_interpolation ? vals.lookup(param) : vals.interpolate(param)
+    cached_env[param] = val
     val
   end
 

data/lib/puppet/ssl/base.rb

@@ -81,7 +81,20 @@ class Puppet::SSL::Base
 
   # Read content from disk appropriately.
   def read(path)
-    @content = wrapped_class.new(File.read(path))
+    # applies to Puppet::SSL::Certificate, Puppet::SSL::CertificateRequest, Puppet::SSL::CertificateRevocationList
+    # Puppet::SSL::Key uses this, but also provides its own override
+    # nothing derives from Puppet::SSL::Certificate, but it is called by a number of other SSL Indirectors:
+    # Puppet::SSL::Certificate::DisabledCa (:find, :save, :destroy)
+    # Puppet::Indirector::CertificateStatus::File (.indirection.find)
+    # Puppet::Network::HTTP::WEBrick (.indirection.find)
+    # Puppet::Network::HTTP::RackREST (.from_instance)
+    # Puppet::Network::HTTP::WEBrickREST (.from_instance)
+    # Puppet::SSL::CertificateAuthority (.new, .indirection.find, .indirection.save)
+    # Puppet::SSL::Host (.indirection.find)
+    # Puppet::SSL::Inventory (.indirection.search, implements its own add / rebuild / serials with encoding UTF8)
+    # Puppet::SSL::CertificateAuthority::Interface (.indirection.find)
+    # Puppet::SSL::Validator::DefaultValidator (.from_instance) / Puppet::SSL::Validator::NoValidator does nothing
+    @content = wrapped_class.new(Puppet::FileSystem.read(path, :encoding => Encoding::ASCII))
   end
 
   # Convert our thing to pem.

data/lib/puppet/ssl/certificate_authority.rb

@@ -173,7 +173,8 @@ class Puppet::SSL::CertificateAuthority
     20.times { pass += (rand(74) + 48).chr }
 
     begin
-      Puppet.settings.setting(:capass).open('w') { |f| f.print pass }
+      # random password is limited to ASCII characters 48 ('0') through 122 ('z')
+      Puppet.settings.setting(:capass).open('w:ASCII') { |f| f.print pass }
     rescue Errno::EACCES => detail
       raise Puppet::Error, "Could not write CA password: #{detail}", detail.backtrace
     end
@@ -216,7 +217,8 @@ class Puppet::SSL::CertificateAuthority
   # file so this one is considered used.
   def next_serial
     serial = 1
-    Puppet.settings.setting(:serial).exclusive_open('a+') do |f|
+    # the serial is 4 hex digits - limited to ASCII
+    Puppet.settings.setting(:serial).exclusive_open('a+:ASCII') do |f|
       f.rewind
       serial = f.read.chomp.hex
       if serial == 0

data/lib/puppet/ssl/configuration.rb

@@ -50,7 +50,10 @@ class Configuration
 
   # read_file makes testing easier.
   def read_file(path)
-    File.read(path)
+    # https://www.ietf.org/rfc/rfc2459.txt defines the x509 V3 certificate format
+    # CA bundles are concatenated X509 certificates, but may also include
+    # comments, which could have UTF-8 characters
+    Puppet::FileSystem.read(path, :encoding => Encoding::UTF_8)
   end
   private :read_file
 end

data/lib/puppet/ssl/inventory.rb

@@ -8,7 +8,10 @@ class Puppet::SSL::Inventory
   # Add a certificate to our inventory.
   def add(cert)
     cert = cert.content if cert.is_a?(Puppet::SSL::Certificate)
-    Puppet.settings.setting(:cert_inventory).open("a") do |f|
+    # RFC 5280 says the cert subject may contain UTF8 - https://www.ietf.org/rfc/rfc5280.txt
+    # Note however that Puppet generated SSL files must only contain ASCII characters
+    # based on the validate_certname method of Puppet::SSL::Base
+    Puppet.settings.setting(:cert_inventory).open('a:UTF-8') do |f|
       f.print format(cert)
     end
   end
@@ -28,7 +31,8 @@ class Puppet::SSL::Inventory
   def rebuild
     Puppet.notice "Rebuilding inventory file"
 
-    Puppet.settings.setting(:cert_inventory).open('w') do |f|
+    # RFC 5280 says the cert subject may contain UTF8 - https://www.ietf.org/rfc/rfc5280.txt
+    Puppet.settings.setting(:cert_inventory).open('w:UTF-8') do |f|
       Puppet::SSL::Certificate.indirection.search("*").each do |cert|
         f.print format(cert.content)
       end
@@ -40,7 +44,10 @@ class Puppet::SSL::Inventory
   def serials(name)
     return [] unless Puppet::FileSystem.exist?(@path)
 
-    File.readlines(@path).collect do |line|
+    # RFC 5280 says the cert subject may contain UTF8 - https://www.ietf.org/rfc/rfc5280.txt
+    # Note however that Puppet generated SSL files must only contain ASCII characters
+    # based on the validate_certname method of Puppet::SSL::Base
+    File.readlines(@path, :encoding => Encoding::UTF_8).collect do |line|
       /^(\S+).+\/CN=#{name}$/.match(line)
     end.compact.map { |m| Integer(m[1]) }
   end

data/lib/puppet/ssl/key.rb

@@ -38,15 +38,19 @@ DOC
   def password
     return nil unless password_file and Puppet::FileSystem.exist?(password_file)
 
-    ::File.read(password_file)
+    # Puppet generates files at the default Puppet[:capass] using ASCII
+    # User configured :passfile could be in any encoding
+    # Use BINARY given the string is passed to an OpenSSL API accepting bytes
+    # note this is only called internally
+    Puppet::FileSystem.read(password_file, :encoding => Encoding::BINARY)
   end
 
   # Optionally support specifying a password file.
   def read(path)
     return super unless password_file
 
-    #@content = wrapped_class.new(::File.read(path), password)
-    @content = wrapped_class.new(::File.read(path), password)
+    # RFC 1421 states PEM is 7-bit ASCII https://tools.ietf.org/html/rfc1421
+    @content = wrapped_class.new(Puppet::FileSystem.read(path, :encoding => Encoding::ASCII), password)
   end
 
   def to_s

data/lib/puppet/test/test_helper.rb

@@ -192,6 +192,9 @@ module Puppet::Test
         $old_env.each {|k, v| Puppet::Util.set_env(k, v, mode) }
       end
 
+      # Clear all environments
+      Puppet.lookup(:environments).clear_all
+
       # Restore the load_path late, to avoid messing with stubs from the test.
       $LOAD_PATH.clear
       $old_load_path.each {|x| $LOAD_PATH << x }

data/lib/puppet/type.rb

@@ -1218,6 +1218,10 @@ class Type
     resource = Puppet::Resource.new(self, title)
     resource.catalog = hash.delete(:catalog)
 
+    if sensitive = hash.delete(:sensitive_parameters)
+      resource.sensitive_parameters = sensitive
+    end
+
     hash.each do |param, value|
       resource[param] = value
     end
@@ -1291,7 +1295,9 @@ class Type
   end
 
   newmetaparam(:audit) do
-    desc "Marks a subset of this resource's unmanaged attributes for auditing. Accepts an
+    desc "(This metaparameter is deprecated and will be ignored in a future release.)
+
+      Marks a subset of this resource's unmanaged attributes for auditing. Accepts an
       attribute name, an array of attribute names, or `all`.
 
       Auditing a resource attribute has two effects: First, whenever a catalog
@@ -1312,6 +1318,12 @@ class Type
       and the second run will log the edit made by Puppet.)"
 
     validate do |list|
+      if Puppet.settings[:strict] != :off
+        # Only warn if `audit` metaparam came from a manifest
+        if file && line
+          puppet_deprecation_warning(_("The `audit` metaparameter is deprecated and will be ignored in a future release."), { :line => line, :file => file })
+        end
+      end
       list = Array(list).collect {|p| p.to_sym}
       unless list == [:all]
         list.each do |param|

data/lib/puppet/type/exec.rb

@@ -157,7 +157,12 @@ module Puppet
         end
 
         unless self.should.include?(@status.exitstatus.to_s)
-          self.fail("#{self.resource[:command]} returned #{@status.exitstatus} instead of one of [#{self.should.join(",")}]")
+          if @resource.parameter(:command).sensitive
+            # Don't print sensitive commands in the clear
+            self.fail("[command redacted] returned #{@status.exitstatus} instead of one of [#{self.should.join(",")}]")
+          else
+            self.fail("'#{self.resource[:command]}' returned #{@status.exitstatus} instead of one of [#{self.should.join(",")}]")
+          end
         end
 
         event
@@ -597,5 +602,15 @@ module Puppet
     def current_username
       Etc.getpwuid(Process.uid).name
     end
+
+    private
+    def set_sensitive_parameters(sensitive_parameters)
+      # Respect sensitive commands
+      if sensitive_parameters.include?(:command)
+        sensitive_parameters.delete(:command)
+        parameter(:command).sensitive = true
+      end
+      super(sensitive_parameters)
+    end
   end
 end

data/lib/puppet/type/group.rb

@@ -79,10 +79,9 @@ module Puppet
     end
 
     newproperty(:members, :array_matching => :all, :required_features => :manages_members) do
-      desc "The members of the group. For directory services where group
-      membership is stored in the group objects, not the users. Use
-      with auth_membership to determine whether the specified members
-      are inclusive or the minimum."
+      desc "The members of the group. For platforms or directory services where group
+        membership is stored in the group objects, not the users. This parameter's
+        behavior can be configured with `auth_membership`."
 
       def change_to_s(currentvalue, newvalue)
         currentvalue = currentvalue.join(",") if currentvalue != :absent
@@ -118,9 +117,12 @@ module Puppet
     end
 
     newparam(:auth_membership, :boolean => true, :parent => Puppet::Parameter::Boolean) do
-      desc "Whether the provider is authoritative for group membership. This
-        must be set to true to allow setting the group to no members with
-        `members => [],`."
+      desc "Configures the behavior of the `members` parameter.
+
+        * `false` (default) --- The provided list of group members is partial,
+          and Puppet **ignores** any members that aren't listed there.
+        * `true` --- The provided list of of group members is comprehensive, and
+          Puppet **purges** any members that aren't listed there."
       defaultto false
     end
 
@@ -146,7 +148,8 @@ module Puppet
     end
 
     newproperty(:attributes, :parent => Puppet::Property::KeyValue, :required_features => :manages_aix_lam) do
-      desc "Specify group AIX attributes in an array of `key=value` pairs."
+      desc "Specify group AIX attributes, as an array of `'key=value'` strings. This
+        parameter's behavior can be configured with `attribute_membership`."
 
       def membership
         :attribute_membership
@@ -162,9 +165,12 @@ module Puppet
     end
 
     newparam(:attribute_membership) do
-      desc "Whether specified attribute value pairs should be treated as the only attributes
-        of the user or whether they should merely
-        be treated as the minimum list."
+      desc "AIX only. Configures the behavior of the `attributes` parameter.
+
+        * `minimum` (default) --- The provided list of attributes is partial, and Puppet
+          **ignores** any attributes that aren't listed there.
+        * `inclusive` --- The provided list of attributes is comprehensive, and
+          Puppet **purges** any attributes that aren't listed there."
 
       newvalues(:inclusive, :minimum)
 

data/lib/puppet/type/user.rb

@@ -739,7 +739,9 @@ module Puppet
     def unknown_keys_in_file(keyfile)
       names = []
       name_index = 0
-      File.new(keyfile).each do |line|
+      # RFC 4716 specifies UTF-8 allowed in public key files per https://www.ietf.org/rfc/rfc4716.txt
+      # the authorized_keys file may contain UTF-8 comments
+      Puppet::FileSystem.open(keyfile, nil, 'r:UTF-8').each do |line|
         next unless line =~ Puppet::Type.type(:ssh_authorized_key).keyline_regex
         # the name is stored in the 4th capture of the regex
         name = $4

data/lib/puppet/util.rb

@@ -10,6 +10,7 @@ require 'puppet/util/platform'
 require 'puppet/util/symbolic_file_mode'
 require 'puppet/file_system/uniquefile'
 require 'securerandom'
+require 'puppet/util/character_encoding'
 
 module Puppet
 module Util

data/lib/puppet/util/character_encoding.rb

@@ -0,0 +1,95 @@
+# A module to centralize heuristics/practices for managing character encoding in Puppet
+
+module Puppet::Util::CharacterEncoding
+  class << self
+    # Warning! This is a destructive method - the string supplied is modified!
+    # @api public
+    # @param [String] string a string to transcode / force_encode to utf-8
+    # @return [String] string if already utf-8, OR
+    #   the same string with external encoding set to utf-8 if bytes are valid utf-8 OR
+    #   the same string transcoded to utf-8 OR
+    #   nil upon a failure to legitimately set external encoding or transcode string
+    def convert_to_utf_8!(string)
+      currently_valid = string.valid_encoding?
+
+      begin
+        if string.encoding == Encoding::UTF_8
+          if currently_valid
+            return string
+          else
+            # If a string is currently believed to be UTF-8, but is also not
+            # valid_encoding?, we have no recourse but to fail because we have no
+            # idea what encoding this string originally came from where it *was*
+            # valid - all we know is it's not currently valid UTF-8.
+            raise EncodingError
+          end
+        elsif valid_utf_8_bytes?(string)
+          # Before we try to transcode the string, check if it is valid UTF-8 as
+          # currently constitued (in its non-UTF-8 encoding), and if it is, limit
+          # ourselves to setting the external encoding of the string to UTF-8
+          # rather than actually transcoding it. We do this to handle
+          # a couple scenarios:
+
+          # The first scenario is that the string was originally valid UTF-8 but
+          # the current puppet run is not in a UTF-8 environment. In this case,
+          # the string will likely have invalid byte sequences (i.e.,
+          # string.valid_encoding? == false), and attempting to transcode will
+          # fail with Encoding::InvalidByteSequenceError, referencing the
+          # invalid byte sequence in the original, pre-transcode, string. We
+          # might have gotten here, for example, if puppet is run first in a
+          # user context with UTF-8 encoding (setting the "is" value to UTF-8)
+          # and then later run via cron without UTF-8 specified, resulting in in
+          # EN_US (ISO-8859-1) on many systems. In this scenario we're
+          # effectively best-guessing this string originated as UTF-8 and only
+          # set external encoding to UTF-8 - transcoding would have failed
+          # anyway.
+
+          # The second scenario (more rare, I expect) is that this string does
+          # NOT have invalid byte sequences (string.valid_encoding? == true),
+          # but is *ALSO valid unicode*.
+          # Our example case is "\u16A0" - "RUNIC LETTER FEHU FEOH FE"
+          # http://www.fileformat.info/info/unicode/char/16A0/index.htm
+          # 0xE1 0x9A 0xA0 / 225 154 160
+          # These bytes are valid in ISO-8859-1 but the character they represent
+          # transcodes cleanly in ruby to *different* characters in UTF-8.
+          # That's not what we want if the user intended the original string as
+          # UTF-8. We can only guess, so if the string is valid UTF-8 as
+          # currently constituted, we default to assuming the string originated
+          # in UTF-8 and do not transcode it - we only set external encoding.
+          return string.force_encoding(Encoding::UTF_8)
+        elsif currently_valid
+          # If the string is not currently valid UTF-8 but it can be transcoded
+          # (it is valid in its current encoding), we can guess this string was
+          # not originally unicode. Transcode it to UTF-8. For strings with
+          # original encodings like SHIFT_JIS, this should be the final result.
+          return string.encode!(Encoding::UTF_8)
+        else
+          # If the string is neither valid UTF-8 as-is nor valid in its current
+          # encoding, fail. It requires user remediation.
+          raise EncodingError
+        end
+      rescue EncodingError => detail
+        # Catch both our own self-determined failure to transcode as well as any
+        # error on ruby's part, ie Encoding::UndefinedConversionError on a
+        # failure to encode!.
+        Puppet.debug(_("%{error}: %{value} is not valid UTF-8 and cannot be transcoded by Puppet.") %
+          { error: detail.inspect, value: string.dump })
+        return nil
+      end
+    end
+
+    private
+
+    # Do our best to determine if a string is valid UTF-8 via String#valid_encoding? without permanently
+    # modifying or duplicating the string due to performance concerns
+    # @api private
+    # @param [String] string a string to test
+    # @return [Boolean] whether we think the string is UTF-8 or not
+    def valid_utf_8_bytes?(string)
+      original_encoding = string.encoding
+      valid = string.force_encoding(Encoding::UTF_8).valid_encoding?
+      string.force_encoding(original_encoding)
+      valid
+    end
+  end
+end