puppet 4.9.4 → 4.10.0

data/lib/puppet/indirector/file_bucket_file/file.rb

@@ -133,9 +133,17 @@ module Puppet::FileBucketFile
       return false
     end
 
-    # @param contents_file [Object] Opaque file path
-    # @param paths_file [Object] Opaque file path
-    #
+    # @param bucket_file [Puppet::FileBucket::File] IO object representing
+    #   content to back up
+    # @param files_original_path [String] Path to original source file on disk
+    # @param contents_file [Pathname] Opaque file path to intended backup
+    #   location
+    # @param paths_file [Pathname] Opaque file path to file containing source
+    #   file paths on disk
+    # @return [void]
+    # @raise [Puppet::FileBucket::BucketError] on possible sum collision between
+    #   existing and new backup
+    # @api private
     def save_to_disk(bucket_file, files_original_path, contents_file, paths_file)
       Puppet::Util.withumask(0007) do
         unless Puppet::FileSystem.dir_exist?(paths_file)
@@ -148,15 +156,28 @@ module Puppet::FileBucketFile
         # same way when the bytes are decoded as UTF-8, continue using system encoding
         Puppet::FileSystem.exclusive_open(paths_file, 0640, 'a+:external') do |f|
           if Puppet::FileSystem.exist?(contents_file)
-            verify_identical_file!(contents_file, bucket_file)
-            Puppet::FileSystem.touch(contents_file)
-          else
-            Puppet::FileSystem.open(contents_file, 0440, 'wb') do |of|
-              # PUP-1044 writes all of the contents
-              bucket_file.stream() do |src|
-                FileUtils.copy_stream(src, of)
-              end
+            if verify_identical_file(contents_file, bucket_file)
+              Puppet.info "FileBucket got a duplicate file #{bucket_file.checksum}"
+              Puppet::FileSystem.touch(contents_file)
+            elsif contents_file_matches_checksum?(contents_file, bucket_file.checksum_data, bucket_file.checksum_type)
+              # If the contents or sizes don't match, but the checksum does,
+              # then we've found a conflict (potential hash collision).
+              # Unlikely, but quite bad. Don't remove the file in case it's
+              # needed, but ask the user to validate.
+              # Note: Don't print the full path to the bucket file in the
+              # exception to avoid disclosing file system layout on server.
+              Puppet.err(_("Unable to verify existing FileBucket backup at '%{path}'.") % { path: contents_file.to_path })
+              raise Puppet::FileBucket::BucketError, _("Existing backup and new file have different content but same checksum, %{value}. Verify existing backup and remove if incorrect.") %
+                { value: bucket_file.checksum }
+            else
+              # PUP-1334 If the contents_file exists but does not match its
+              # checksum, our backup has been corrupted. Warn about overwriting
+              # it, and proceed with new backup.
+              Puppet.warning(_("Existing backup does not match its expected sum, %{sum}. Overwriting corrupted backup.") % { sum: bucket_file.checksum })
+              copy_bucket_file_to_contents_file(contents_file, bucket_file)
             end
+          else
+            copy_bucket_file_to_contents_file(contents_file, bucket_file)
           end
 
           unless path_match(f, files_original_path)
@@ -189,19 +210,45 @@ module Puppet::FileBucketFile
       Puppet::FileSystem.pathname(subfile ? ::File.join(basedir, subfile) : basedir)
     end
 
-    # @param contents_file [Object] Opaque file path
-    # @param bucket_file [IO]
-    def verify_identical_file!(contents_file, bucket_file)
-      if bucket_file.size == Puppet::FileSystem.size(contents_file)
-        if bucket_file.stream() {|s| Puppet::FileSystem.compare_stream(contents_file, s) }
-          Puppet.info "FileBucket got a duplicate file #{bucket_file.checksum}"
-          return
+    # @param contents_file [Pathname] Opaque file path to intended backup
+    #   location
+    # @param bucket_file [Puppet::FileBucket::File] IO object representing
+    #   content to back up
+    # @return [Boolean] whether the data in contents_file is of the same size
+    #   and content as that in the bucket_file
+    # @api private
+    def verify_identical_file(contents_file, bucket_file)
+      (bucket_file.size == Puppet::FileSystem.size(contents_file)) &&
+        (bucket_file.stream() {|s| Puppet::FileSystem.compare_stream(contents_file, s) })
+    end
+
+    # @param contents_file [Pathname] Opaque file path to intended backup
+    #   location
+    # @param expected_checksum_data [String] expected value of checksum of type
+    #   checksum_type
+    # @param checksum_type [String] type of check sum of checksum_data, ie "md5"
+    # @return [Boolean] whether the checksum of the contents_file matches the
+    #   supplied checksum
+    # @api private
+    def contents_file_matches_checksum?(contents_file, expected_checksum_data, checksum_type)
+      contents_file_checksum_data = Puppet::Util::Checksums.method(:"#{checksum_type}_file").call(contents_file.to_path)
+      contents_file_checksum_data == expected_checksum_data
+    end
+
+    # @param contents_file [Pathname] Opaque file path to intended backup
+    #   location
+    # @param bucket_file [Puppet::FileBucket::File] IO object representing
+    #   content to back up
+    # @return [void]
+    # @api private
+    def copy_bucket_file_to_contents_file(contents_file, bucket_file)
+      Puppet::FileSystem.open(contents_file, 0440, 'wb') do |of|
+        # PUP-1044 writes all of the contents
+        bucket_file.stream() do |src|
+          FileUtils.copy_stream(src, of)
         end
       end
-
-      # If the contents or sizes don't match, then we've found a conflict.
-      # Unlikely, but quite bad.
-      raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}"
     end
+
   end
 end

data/lib/puppet/indirector/key/file.rb

@@ -39,7 +39,8 @@ class Puppet::SSL::Key::File < Puppet::Indirector::SslFile
     super
 
     begin
-      Puppet.settings.setting(:publickeydir).open_file(public_key_path(request.key), 'w') do |f|
+      # RFC 1421 states PEM is 7-bit ASCII https://tools.ietf.org/html/rfc1421
+      Puppet.settings.setting(:publickeydir).open_file(public_key_path(request.key), 'w:ASCII') do |f|
         f.print request.instance.content.public_key.to_pem
       end
     rescue => detail

data/lib/puppet/indirector/ssl_file.rb

@@ -112,6 +112,8 @@ class Puppet::Indirector::SslFile < Puppet::Indirector::Terminus
 
   def create_model(name, path)
     result = model.new(name)
+    # calls Puppet::SSL::Base#read for subclasses of Puppet::SSL::Base
+    # with the exception of any overrides, like Puppet::SSL::Key
     result.read(path)
     result
   end
@@ -158,13 +160,32 @@ class Puppet::Indirector::SslFile < Puppet::Indirector::Terminus
   # Yield a filehandle set up appropriately, either with our settings doing
   # the work or opening a filehandle manually.
   def write(name, path)
+    # All types serialized to disk contain only ASCII content:
+    # * SSL::Key may be a .export(OpenSSL::Cipher::DES.new(:EDE3, :CBC), pass) or .to_pem
+    # * All other classes are translated to strings by calling .to_pem
+
+    # Serialization of:
+    # Puppet::SSL::Certificate by Puppet::SSL::Certificate::Ca, Puppet::SSL::Certificate::File
+    # -----BEGIN CERTIFICATE-----
+    # Puppet::SSL::Key by Puppet::SSL::Key::Ca, Puppet::SSL::Key::File
+    # -----BEGIN RSA PRIVATE KEY-----
     if ca?(name) and ca_location
-      Puppet.settings.setting(self.class.ca_setting).open('w') { |f| yield f }
+      Puppet.settings.setting(self.class.ca_setting).open('w:ASCII') { |f| yield f }
+    # Serialization of:
+    # Puppet::SSL::CertificateRevocationList by Puppet::SSL::CertificateRevocationList::Ca, Puppet::SSL::CertificateRevocationList::File
+    # -----BEGIN X509 CRL-----
     elsif file_location
-      Puppet.settings.setting(self.class.file_setting).open('w') { |f| yield f }
+      Puppet.settings.setting(self.class.file_setting).open('w:ASCII') { |f| yield f }
+    # Serialization of:
+    # Puppet::SSL::Certificate by Puppet::SSL::Certificate::Ca, Puppet::SSL::Certificate::File
+    # -----BEGIN CERTIFICATE-----
+    # Puppet::SSL::CertificateRequest by Puppet::SSL::CertificateRequest::Ca, Puppet::SSL::CertificateRequest::File
+    # -----BEGIN CERTIFICATE REQUEST-----
+    # Puppet::SSL::Key by Puppet::SSL::Key::Ca, Puppet::SSL::Key::File
+    # -----BEGIN RSA PRIVATE KEY-----
     elsif setting = self.class.directory_setting
       begin
-        Puppet.settings.setting(setting).open_file(path, 'w') { |f| yield f }
+        Puppet.settings.setting(setting).open_file(path, 'w:ASCII') { |f| yield f }
       rescue => detail
         raise Puppet::Error, "Could not write #{path} to #{setting}: #{detail}", detail.backtrace
       end

data/lib/puppet/module.rb

@@ -65,8 +65,7 @@ class Puppet::Module
     @environment = environment
 
     assert_validity
-
-    load_metadata if has_metadata?
+    load_metadata
 
     @absolute_path_to_manifests = Puppet::FileSystem::PathPattern.absolute(manifests)
   end
@@ -86,24 +85,12 @@ class Puppet::Module
   end
 
   def has_metadata?
-    return false unless metadata_file
-
-    return false unless Puppet::FileSystem.exist?(metadata_file)
-
     begin
-      metadata =  JSON.parse(File.read(metadata_file, :encoding => 'utf-8'))
-    rescue JSON::JSONError => e
-      msg = "#{name} has an invalid and unparsable metadata.json file. The parse error: #{e.message}"
-      case Puppet[:strict]
-      when :off, :warning
-        Puppet.warning(msg)
-      when :error
-        raise FaultyMetadata, msg
-      end
-      return false
+      load_metadata
+      @metadata.is_a?(Hash) && !@metadata.empty?
+    rescue Puppet::Module::MissingMetadata
+      false
     end
-
-    return metadata.is_a?(Hash) && !metadata.keys.empty?
   end
 
   FILETYPES.each do |type, location|
@@ -151,14 +138,33 @@ class Puppet::Module
     @license_file = File.join(path, "License")
   end
 
+  def read_metadata
+    md_file = metadata_file
+    md_file.nil? ? {} : JSON.parse(File.read(md_file, :encoding => 'utf-8'))
+  rescue Errno::ENOENT
+    {}
+  rescue JSON::JSONError => e
+    msg = "#{name} has an invalid and unparsable metadata.json file. The parse error: #{e.message}"
+    case Puppet[:strict]
+    when :off, :warning
+      Puppet.warning(msg)
+    when :error
+      raise FaultyMetadata, msg
+    end
+    {}
+  end
+
   def load_metadata
-    @metadata = data = JSON.parse(File.read(metadata_file, :encoding => 'utf-8'))
+    return if instance_variable_defined?(:@metadata)
+
+    @metadata = data = read_metadata
+    return if data.empty?
+
     @forge_name = data['name'].gsub('-', '/') if data['name']
 
     [:source, :author, :version, :license, :dependencies].each do |attr|
-      unless value = data[attr.to_s]
-        raise MissingMetadata, "No #{attr} module metadata provided for #{self.name}"
-      end
+      value = data[attr.to_s]
+      raise MissingMetadata, "No #{attr} module metadata provided for #{self.name}" if value.nil?
 
       if attr == :dependencies
         unless value.is_a?(Array)

data/lib/puppet/network/http/compression.rb

@@ -20,7 +20,8 @@ module Puppet::Network::HTTP::Compression
     def uncompress_body(response)
       case response['content-encoding']
       when 'gzip'
-        return Zlib::GzipReader.new(StringIO.new(response.body)).read
+        # ZLib::GzipReader has an associated encoding, by default Encoding.default_external
+        return Zlib::GzipReader.new(StringIO.new(response.body), :encoding => Encoding::BINARY).read
       when 'deflate'
         return Zlib::Inflate.new.inflate(response.body)
       when nil, 'identity'

data/lib/puppet/parser/compiler.rb

@@ -217,7 +217,7 @@ class Puppet::Parser::Compiler
     {
       :current_environment => environment,
       :global_scope => @topscope,             # 4x placeholder for new global scope
-      :loaders  => lambda {|| loaders() },    # 4x loaders
+      :loaders  => @loaders,                  # 4x loaders
       :injector => lambda {|| injector() }    # 4x API - via context instead of via compiler
     }
   end
@@ -357,7 +357,7 @@ class Puppet::Parser::Compiler
     end
   end
 
-  # Evaluates each specified class in turn. If there are any classes that 
+  # Evaluates each specified class in turn. If there are any classes that
   # can't be found, an error is raised. This method really just creates resource objects
   # that point back to the classes, and then the resources are themselves
   # evaluated later in the process.
@@ -442,10 +442,6 @@ class Puppet::Parser::Compiler
     @injector
   end
 
-  def loaders
-    @loaders ||= Puppet::Pops::Loaders.new(environment)
-  end
-
   def boot_injector
     create_boot_injector(nil) if @boot_injector.nil?
     @boot_injector
@@ -752,6 +748,9 @@ class Puppet::Parser::Compiler
     # Create the initial scope, it is needed early
     @topscope = Puppet::Parser::Scope.new(self)
 
+    # Initialize loaders and Pcore
+    @loaders = Puppet::Pops::Loaders.new(environment)
+
     # Need to compute overrides here, and remember them, because we are about to
     # enter the magic zone of known_resource_types and initial import.
     # Expensive entries in the context are bound lazily.
@@ -852,47 +851,25 @@ class Puppet::Parser::Compiler
     @topscope.set_facts(facts_hash)
   end
 
+  SETTINGS = 'settings'.freeze
+
   def create_settings_scope
-    settings_type = Puppet::Resource::Type.new :hostclass, "settings"
-    environment.known_resource_types.add(settings_type)
+    resource_types = environment.known_resource_types
+    settings_type = resource_types.hostclass(SETTINGS)
+    if settings_type.nil?
+      settings_type = Puppet::Resource::Type.new(:hostclass, SETTINGS)
+      resource_types.add(settings_type)
+    end
 
-    settings_resource = Puppet::Parser::Resource.new("class", "settings", :scope => @topscope)
+    settings_resource = Puppet::Parser::Resource.new('class', SETTINGS, :scope => @topscope)
 
     @catalog.add_resource(settings_resource)
 
     settings_type.evaluate_code(settings_resource)
 
     scope = @topscope.class_scope(settings_type)
-
-    env = environment
-    settings_hash = {}
-    Puppet.settings.each do |name, setting|
-      next if name == :name
-      s_name = name.to_s
-      # Construct a hash (in anticipation it will be set in top scope under a name like $settings)
-      settings_hash[s_name] = transform_setting(env[name])
-      scope[s_name] = settings_hash[s_name]
-    end
-  end
-
-  def transform_setting(val)
-    case val
-    when Integer, Float, String, TrueClass, FalseClass, NilClass
-      val
-    when Symbol
-      val == :undef ? nil : val.to_s
-    when Array
-      val.map {|entry| transform_setting(entry) }
-    when Hash
-      result = {}
-      val.each {|k,v| result[transform_setting(k)] = transform_setting(v) }
-      result
-    else
-      # not ideal, but required as there are settings values that are special
-      val.to_s
-    end
+    scope.merge_settings(environment.name)
   end
-  private :transform_setting
 
   # Return an array of all of the unevaluated resources.  These will be definitions,
   # which need to get evaluated into native resources.

data/lib/puppet/parser/functions/hiera.rb

@@ -78,7 +78,7 @@ The returned value's data type depends on the types of the results. In the examp
 above, Hiera matches the 'users' key and returns it as a hash.
 
 The `hiera` function is deprecated in favor of using `lookup` and will be removed in 6.0.0.
-See  https://docs.puppet.com/puppet/#{Puppet.version}/reference/deprecated_language.html.
+See  https://docs.puppet.com/puppet/#{Puppet.minor_version}/reference/deprecated_language.html.
 Replace the calls as follows:
 
 | from  | to |

data/lib/puppet/parser/functions/hiera_array.rb

@@ -66,7 +66,7 @@ $allusers = hiera_array('users') | $key | { "Key \'${key}\' not found" }
 value is a hash, Puppet raises a type mismatch error.
 
 `hiera_array` is deprecated in favor of using `lookup` and will be removed in 6.0.0.
-See  https://docs.puppet.com/puppet/#{Puppet.version}/reference/deprecated_language.html.
+See  https://docs.puppet.com/puppet/#{Puppet.minor_version}/reference/deprecated_language.html.
 Replace the calls as follows:
 
 | from  | to |

data/lib/puppet/parser/functions/hiera_hash.rb

@@ -76,7 +76,7 @@ $allusers = hiera_hash('users') | $key | { "Key \'${key}\' not found" }
 found in the data sources are strings or arrays, Puppet raises a type mismatch error.
 
 `hiera_hash` is deprecated in favor of using `lookup` and will be removed in 6.0.0.
-See  https://docs.puppet.com/puppet/#{Puppet.version}/reference/deprecated_language.html.
+See  https://docs.puppet.com/puppet/#{Puppet.minor_version}/reference/deprecated_language.html.
 Replace the calls as follows:
 
 | from  | to |

data/lib/puppet/parser/functions/hiera_include.rb

@@ -76,7 +76,7 @@ hiera_include('classes') | $key | {"Key \'${key}\' not found" }
 ~~~
 
 `hiera_include` is deprecated in favor of using a combination of `include`and `lookup` and will be
-removed in 6.0.0. See  https://docs.puppet.com/puppet/#{Puppet.version}/reference/deprecated_language.html.
+removed in 6.0.0. See  https://docs.puppet.com/puppet/#{Puppet.minor_version}/reference/deprecated_language.html.
 Replace the calls as follows:
 
 | from  | to |

data/lib/puppet/parser/scope.rb

@@ -588,7 +588,7 @@ class Puppet::Parser::Scope
   end
 
   def is_topscope?
-    compiler and self == compiler.topscope
+    @compiler && equal?(@compiler.topscope)
   end
 
   def lookup_qualified_variable(class_name, variable_name, position)
@@ -710,14 +710,45 @@ class Puppet::Parser::Scope
     }
   end
 
-  RESERVED_VARIABLE_NAMES = ['trusted', 'facts'].freeze
+  # Merge all settings for the given _env_name_ into this scope
+  # @param env_name [Symbol] the name of the environment
+  def merge_settings(env_name)
+    settings = Puppet.settings
+    table = effective_symtable(false)
+    settings.each_key do |name|
+      next if :name == name
+      table[name.to_s] = transform_setting(settings.value_sym(name, env_name))
+    end
+    nil
+  end
+
+  def transform_setting(val)
+    if val.is_a?(String) || val.is_a?(Numeric) || true == val || false == val || nil == val
+      val
+    elsif val.is_a?(Array)
+      val.map {|entry| transform_setting(entry) }
+    elsif val.is_a?(Hash)
+      result = {}
+      val.each {|k,v| result[transform_setting(k)] = transform_setting(v) }
+      result
+    else
+      # not ideal, but required as there are settings values that are special
+      :undef == val ? nil : val.to_s
+    end
+  end
+  private :transform_setting
+
+  VARNAME_TRUSTED = 'trusted'.freeze
+  VARNAME_FACTS = 'facts'.freeze
+  VARNAME_SERVER_FACTS = 'server_facts'.freeze
+  RESERVED_VARIABLE_NAMES = [VARNAME_TRUSTED, VARNAME_FACTS].freeze
 
   # Set a variable in the current scope.  This will override settings
   # in scopes above, but will not allow variables in the current scope
   # to be reassigned.
   #   It's preferred that you use self[]= instead of this; only use this
   # when you need to set options.
-  def setvar(name, value, options = {})
+  def setvar(name, value, options = EMPTY_HASH)
     if name =~ /^[0-9]+$/
       raise Puppet::ParseError.new("Cannot assign to a numeric match result variable '$#{name}'") # unless options[:ephemeral]
     end
@@ -726,12 +757,12 @@ class Puppet::Parser::Scope
     end
 
     # Check for reserved variable names
-    if !options[:privileged] && RESERVED_VARIABLE_NAMES.include?(name)
+    if (name == VARNAME_TRUSTED || name == VARNAME_FACTS) && !options[:privileged]
       raise Puppet::ParseError, "Attempt to assign to a reserved variable name: '#{name}'"
     end
 
     # Check for server_facts reserved variable name if the trusted_sever_facts setting is true
-    if Puppet[:trusted_server_facts] && name == 'server_facts' && !options[:privileged]
+    if name == VARNAME_SERVER_FACTS && !options[:privileged] && Puppet[:trusted_server_facts]
       raise Puppet::ParseError, "Attempt to assign to a reserved variable name: '#{name}'"
     end
 
@@ -798,15 +829,13 @@ class Puppet::Parser::Scope
   #
   # @param use_ephemeral [Boolean] whether the top most ephemeral (of any kind) should be used or not
   def effective_symtable(use_ephemeral)
-    s = @ephemeral.last
-    if use_ephemeral
-      return s || @symtable
-    else
-      while s && !s.is_local_scope?()
-        s = s.parent
-      end
-      s || @symtable
+    s = @ephemeral[-1]
+    return s || @symtable if use_ephemeral
+
+    while s && !s.is_local_scope?()
+      s = s.parent
     end
+    s || @symtable
   end
 
   # Sets the variable value of the name given as an argument to the given value. The value is
@@ -816,12 +845,12 @@ class Puppet::Parser::Scope
   #
   # @param varname [String] The variable name to which the value is assigned. Must not contain `::`
   # @param value [String] The value to assign to the given variable name.
-  # @param options [Hash] Additional options, not part of api.
+  # @param options [Hash] Additional options, not part of api and no longer used.
   #
   # @api public
   #
-  def []=(varname, value, options = {})
-    setvar(varname, value, options = {})
+  def []=(varname, value, _ = nil)
+    setvar(varname, value)
   end
 
   def append_value(bound_value, new_value)
@@ -901,6 +930,19 @@ class Puppet::Parser::Scope
     find_global_scope.without_ephemeral_scopes(&block)
   end
 
+  # Execute given block with a ephemeral scope containing the given variables
+  # @api private
+  def with_local_scope(scope_variables)
+    local = LocalScope.new(@ephemeral.last)
+    scope_variables.each_pair { |k, v| local[k] = v }
+    @ephemeral.push(local)
+    begin
+      yield(self)
+    ensure
+      @ephemeral.pop
+    end
+  end
+
   # Execute given block with all ephemeral popped from the ephemeral stack
   #
   # @api private
@@ -959,7 +1001,7 @@ class Puppet::Parser::Scope
       new_ephemeral(true)
       match.each {|k,v| setvar(k, v, :file => file, :line => line, :ephemeral => true) }
       # Must always have an inner match data scope (that starts out as transparent)
-      # In 3x slightly wasteful, since a new nested scope is created for a match 
+      # In 3x slightly wasteful, since a new nested scope is created for a match
       # (TODO: Fix that problem)
       new_ephemeral(false)
     else