puppet 6.13.0-x64-mingw32 → 6.18.0-x64-mingw32

data/lib/puppet/transaction.rb

@@ -202,7 +202,7 @@ class Puppet::Transaction
     # mark the end of transaction evaluate.
     report.transaction_completed = true
 
-    Puppet.debug "Finishing transaction #{object_id}"
+    Puppet.debug { "Finishing transaction #{object_id}" }
   end
 
   # Wraps application run state check to flag need to interrupt processing
@@ -373,7 +373,7 @@ class Puppet::Transaction
     type_name = provider_class.resource_type.name
     return if @prefetched_providers[type_name][provider_class.name] ||
       @prefetch_failed_providers[type_name][provider_class.name]
-    Puppet.debug "Prefetching #{provider_class.name} resources for #{type_name}"
+    Puppet.debug { "Prefetching #{provider_class.name} resources for #{type_name}" }
     begin
       provider_class.prefetch(resources)
     rescue LoadError, Puppet::MissingCommand => detail

data/lib/puppet/transaction/persistence.rb

@@ -62,7 +62,7 @@ class Puppet::Transaction::Persistence
     result = nil
     Puppet::Util.benchmark(:debug, _("Loaded transaction store file in %{seconds} seconds")) do
       begin
-        result = Puppet::Util::Yaml.safe_load_file(filename, [Symbol])
+        result = Puppet::Util::Yaml.safe_load_file(filename, [Symbol, Time])
       rescue Puppet::Util::Yaml::YamlLoadError => detail
         Puppet.log_exception(detail, _("Transaction store file %{filename} is corrupt (%{detail}); replacing") % { filename: filename, detail: detail })
 

data/lib/puppet/transaction/report.rb

@@ -122,7 +122,7 @@ class Puppet::Transaction::Report
 
   # @!attribute [r] corrective_change
   #   @return [Boolean] true if the report contains any events and resources that had
-  #      corrective changes.
+  #      corrective changes, including noop corrective changes.
   attr_reader :corrective_change
 
   # @return [Boolean] true if one or more resources attempted to generate
@@ -217,13 +217,13 @@ class Puppet::Transaction::Report
   end
 
   # @api private
-  def initialize(configuration_version=nil, environment=nil, transaction_uuid=nil, job_id=nil)
+  def initialize(configuration_version=nil, environment=nil, transaction_uuid=nil, job_id=nil, start_time=Time.now)
     @metrics = {}
     @logs = []
     @resource_statuses = {}
     @external_times ||= {}
     @host = Puppet[:node_name_value]
-    @time = Time.now
+    @time = start_time
     @report_format = 10
     @puppet_version = Puppet.version
     @configuration_version = configuration_version

data/lib/puppet/trusted_external.rb

@@ -3,11 +3,39 @@ module Puppet::TrustedExternal
   def retrieve(certname)
     command = Puppet[:trusted_external_command]
     return nil unless command
+    Puppet.debug { _("Retrieving trusted external data from %{command}") % {command: command} }
+    setting_type = Puppet.settings.setting(:trusted_external_command).type
+    if setting_type == :file
+      return fetch_data(command, certname)
+    end
+    # command is a directory. Thus, data is a hash of <basename> => <data> for
+    # each executable file in command. For example, if the files 'servicenow.rb',
+    # 'unicorn.sh' are in command, then data is the following hash:
+    #   { 'servicenow' => <servicenow.rb output>, 'unicorn' => <unicorn.sh output> }
+    data = {}
+    Puppet::FileSystem.children(command).each do |file|
+      abs_path = Puppet::FileSystem.expand_path(file)
+      executable_file = Puppet::FileSystem.file?(abs_path) && Puppet::FileSystem.executable?(abs_path)
+      unless executable_file
+        Puppet.debug { _("Skipping non-executable file %{file}")  % { file: abs_path } }
+        next
+      end
+      basename = file.basename(file.extname).to_s
+      unless data[basename].nil?
+        raise Puppet::Error, _("There is more than one '%{basename}' script in %{dir}") % { basename: basename, dir: command }
+      end
+      data[basename] = fetch_data(abs_path, certname)
+    end
+    data
+  end
+  module_function :retrieve
+
+  def fetch_data(command, certname)
     result = Puppet::Util::Execution.execute([command, certname], {
       :combine => false,
       :failonfail => true,
     })
     JSON.parse(result)
   end
-  module_function :retrieve
+  module_function :fetch_data
 end

data/lib/puppet/type.rb

@@ -10,6 +10,7 @@ require 'puppet/metatype/manager'
 require 'puppet/util/errors'
 require 'puppet/util/logging'
 require 'puppet/util/tagging'
+require 'puppet/concurrent/lock'
 
 # see the bottom of the file for the rest of the inclusions
 
@@ -84,6 +85,11 @@ class Type
   # Comparing type instances.
   include Comparable
 
+  # These variables are used in Metatype::Manager for managing types
+  @types = {}
+  @manager_lock = Puppet::Concurrent::Lock.new
+  extend Puppet::MetaType::Manager
+
   # Compares this type against the given _other_ (type) and returns -1, 0, or +1 depending on the order.
   # @param other [Object] the object to compare against (produces nil, if not kind of Type}
   # @return [-1, 0, +1, nil] produces -1 if this type is before the given _other_ type, 0 if equals, and 1 if after.
@@ -110,8 +116,10 @@ class Type
 
   # Allow declaring that a type is actually a capability
   class << self
+    # @deprecated application orchestration will be removed in puppet 7
     attr_accessor :is_capability
 
+    # @deprecated application orchestration will be removed in puppet 7
     def is_capability?
       c = is_capability
       c.nil? ? false : c
@@ -123,6 +131,8 @@ class Type
   # represent application instances, this implementation always returns
   # +false+. Having this method though makes code checking whether a
   # resource is an application instance simpler
+  #
+  # @deprecated application orchestration will be removed in puppet 7
   def self.application?
       false
   end
@@ -743,7 +753,7 @@ class Type
   # @param options [Hash] options merged with a fixed set of options defined by this method, passed on to {Puppet::Transaction::Event}.
   # @return [Puppet::Transaction::Event] the created event
   def event(options = {})
-    Puppet::Transaction::Event.new({:resource => self, :file => file, :line => line, :tags => tags}.merge(options))
+    Puppet::Transaction::Event.new(**{:resource => self, :file => file, :line => line, :tags => tags}.merge(options))
   end
 
   # @return [Object, nil] Returns the 'should' (wanted state) value for a specified property, or nil if the
@@ -1199,15 +1209,17 @@ class Type
       provider.instances.collect do |instance|
         # We always want to use the "first" provider instance we find, unless the resource
         # is already managed and has a different provider set
-        other = provider_instances[instance.name]
+        title = instance.respond_to?(:title) ? instance.title : instance.name
+        other = provider_instances[title]
         if other
-          Puppet.debug "%s %s found in both %s and %s; skipping the %s version" %
-            [self.name.to_s.capitalize, instance.name, other.class.name, instance.class.name, instance.class.name]
+          Puppet.debug {
+            "%s %s found in both %s and %s; skipping the %s version" % [self.name.to_s.capitalize, title, other.class.name, instance.class.name, instance.class.name]
+          }
           next
         end
-        provider_instances[instance.name] = instance
+        provider_instances[title] = instance
 
-        result = new(:name => instance.name, :provider => instance)
+        result = new(:name => instance.name, :provider => instance, :title => title)
         properties.each { |name| result.newattr(name) }
         result
       end
@@ -1708,6 +1720,7 @@ class Type
     }
   end
 
+  # @deprecated application orchestration will be removed in puppet 7
   newmetaparam(:export, :parent => RelationshipMetaparam, :attributes => {:direction => :out, :events => :NONE}) do
           desc <<EOS
 Export a capability resource.
@@ -1733,6 +1746,7 @@ web { server:
 EOS
   end
 
+  # @deprecated application orchestration will be removed in puppet 7
   newmetaparam(:consume, :parent => RelationshipMetaparam, :attributes => {:direction => :in, :events => :NONE}) do
           desc <<EOS
 Consume a capability resource.
@@ -1882,7 +1896,7 @@ end
     name = name.intern
 
     if unprovide(name)
-      Puppet.debug "Reloading #{name} #{self.name} provider"
+      Puppet.debug { "Reloading #{name} #{self.name} provider" }
     end
 
     pname = options[:parent]
@@ -2284,7 +2298,6 @@ end
     #
     attr_accessor :self_refresh
     include Enumerable, Puppet::Util::ClassGen
-    include Puppet::MetaType::Manager
 
     include Puppet::Util
     include Puppet::Util::Logging

data/lib/puppet/type/file.rb

@@ -110,6 +110,19 @@ Puppet::Type.newtype(:file) do
       balancer to direct all filebucket traffic to a single master, or use
       something like an out-of-band rsync task to synchronize the content on all
       masters.
+
+      > **Note**: Enabling and using the backup option, and by extension the 
+        filebucket resource, requires appropriate planning and management to ensure 
+        that sufficient disk space is available for the file backups. Generally, you 
+        can implement this using one of the following two options:
+        - Use a `find` command and `crontab` entry to retain only the last X days 
+        of file backups. For example:
+        
+        ```
+        find /opt/puppetlabs/server/data/puppetserver/bucket -type f -mtime +45 -atime +45 -print0 | xargs -0 rm
+        ```
+        
+        - Restrict the directory to a maximum size after which the oldest items are removed.
     EOT
 
     defaultto "puppet"
@@ -388,8 +401,12 @@ Puppet::Type.newtype(:file) do
     end
   end
 
-  CREATORS = [:content, :source, :target]
-  SOURCE_ONLY_CHECKSUMS = [:none, :ctime, :mtime]
+  # mutually exclusive ways to create files
+  CREATORS = [:content, :source, :target].freeze
+
+  # This is both "checksum types that can't be used with the content property"
+  # and "checksum types that are not digest based"
+  SOURCE_ONLY_CHECKSUMS = [:none, :ctime, :mtime].freeze
 
   validate do
     creator_count = 0
@@ -415,7 +432,7 @@ Puppet::Type.newtype(:file) do
       @parameters[:content].value = @parameters[:checksum].sum(@parameters[:content].actual_content)
     end
 
-    if self[:checksum] && self[:checksum_value] && !send("#{self[:checksum]}?", self[:checksum_value])
+    if self[:checksum] && self[:checksum_value] && !valid_checksum?(self[:checksum], self[:checksum_value])
       self.fail _("Checksum value '%{value}' is not a valid checksum type %{checksum}") % { value: self[:checksum_value], checksum: self[:checksum] }
     end
 
@@ -917,7 +934,7 @@ Puppet::Type.newtype(:file) do
           # that out.
         end
 
-        fail_if_checksum_is_wrong(file.path, content_checksum) if validate_checksum?
+        fail_if_checksum_is_wrong(property, file.path, content_checksum)
       end
     else
       umask = mode ? 000 : 022
@@ -1027,17 +1044,38 @@ Puppet::Type.newtype(:file) do
     end
   end
 
-  # Should we validate the checksum of the file we're writing?
-  def validate_checksum?
-    self[:checksum] !~ /time/
-  end
-
   # Make sure the file we wrote out is what we think it is.
-  def fail_if_checksum_is_wrong(path, content_checksum)
-    newsum = parameter(:checksum).sum_file(path)
-    return if [:absent, nil, content_checksum].include?(newsum)
+  # @param [Puppet::Parameter] property the param or property that wrote the file, or nil
+  # @param [String] path to the file
+  # @param [String] the checksum for the local file
+  #
+  # @api private
+  #
+  def fail_if_checksum_is_wrong(property, path, content_checksum)
+    desired_checksum = desired_checksum(property, path)
 
-    self.fail _("File written to disk did not match checksum; discarding changes (%{content_checksum} vs %{newsum})") % { content_checksum: content_checksum, newsum: newsum }
+    if desired_checksum && content_checksum != desired_checksum
+      self.fail _("File written to disk did not match desired checksum; discarding changes (%{content_checksum} vs %{desired_checksum})") % { content_checksum: content_checksum, desired_checksum: desired_checksum }
+    end
+  end
+
+  # Return the desired checksum or nil
+  def desired_checksum(property, path)
+    return if SOURCE_ONLY_CHECKSUMS.include?(self[:checksum])
+
+    if self[:checksum] && self[:checksum_value]
+      "{#{self[:checksum]}}#{self[:checksum_value]}"
+    elsif property && property.name == :source
+      meta = property.metadata
+      return unless meta
+
+      # due to HttpMetadata the checksum type may fallback to mtime, so recheck
+      return if SOURCE_ONLY_CHECKSUMS.include?(meta.checksum_type)
+      meta.checksum
+    elsif property && property.name == :content
+      str = property.actual_content
+      str ? parameter(:checksum).sum(str) : nil
+    end
   end
 
   def write_temporary_file?

data/lib/puppet/type/file/checksum.rb

@@ -9,7 +9,7 @@ Puppet::Type.type(:file).newparam(:checksum) do
 
     The default checksum type is md5."
 
-  newvalues "md5", "md5lite", "sha224", "sha256", "sha256lite", "sha384", "sha512", "mtime", "ctime", "none"
+  newvalues(*Puppet::Util::Checksums.known_checksum_types)
 
   defaultto do
     Puppet[:digest_algorithm].to_sym
@@ -23,18 +23,18 @@ Puppet::Type.type(:file).newparam(:checksum) do
 
   def sum(content)
     content = content.is_a?(Puppet::Pops::Types::PBinaryType::Binary) ? content.binary_buffer : content
-    type = digest_algorithm()
+    type = digest_algorithm
     "{#{type}}" + send(type, content)
   end
 
   def sum_file(path)
-    type = digest_algorithm()
+    type = digest_algorithm
     method = type.to_s + "_file"
     "{#{type}}" + send(method, path).to_s
   end
 
   def sum_stream(&block)
-    type = digest_algorithm()
+    type = digest_algorithm
     method = type.to_s + "_stream"
     checksum = send(method, &block)
     "{#{type}}#{checksum}"

data/lib/puppet/type/file/source.rb

@@ -2,10 +2,7 @@ require 'puppet/file_serving/content'
 require 'puppet/file_serving/metadata'
 require 'puppet/file_serving/terminus_helper'
 
-require 'puppet/util/http_proxy'
-require 'puppet/network/http'
-require 'puppet/network/http/api/indirected_routes'
-require 'puppet/network/http/compression'
+require 'puppet/http'
 
 module Puppet
   # Copy files from a local or remote source.  This state *only* does any work
@@ -14,11 +11,6 @@ module Puppet
   # this state, during retrieval, modifies the appropriate other states
   # so that things get taken care of appropriately.
   Puppet::Type.type(:file).newparam(:source) do
-    include Puppet::Network::HTTP::Compression.module
-
-    BINARY_MIME_TYPES = [
-      Puppet::Network::FormatHandler.format_for('binary').mime
-    ].join(', ').freeze
 
     attr_accessor :source, :local
     desc <<-'EOT'
@@ -31,7 +23,7 @@ module Puppet
       * Fully qualified paths to locally available files (including files on NFS
       shares or Windows mapped drives).
       * `file:` URIs, which behave the same as local file paths.
-      * `http:` URIs, which point to files served by common web servers.
+      * `http(s):` URIs, which point to files served by common web servers.
 
       The normal form of a `puppet:` URI is:
 
@@ -52,9 +44,26 @@ module Puppet
       because HTTP servers do not transfer any metadata that translates to
       ownership or permission details.
 
-      The `http` source uses the server `Content-MD5` header as a checksum to
-      determine if the remote file has changed. If the server response does not
-      include that header, Puppet defaults to using the `Last-Modified` header.
+      Puppet determines if file content is synchronized by computing a checksum
+      for the local file and comparing it against the `checksum_value`
+      parameter. If the `checksum_value` parameter is not specified for
+      `puppet` and `file` sources, Puppet computes a checksum based on its
+      `Puppet[:digest_algorithm]`. For `http(s)` sources, Puppet uses the
+      first HTTP header it recognizes out of the following list:
+      `X-Checksum-Sha256`, `X-Checksum-Sha1`, `X-Checksum-Md5` or `Content-MD5`.
+      If the server response does not include one of these headers, Puppet
+      defaults to using the `Last-Modified` header. Puppet updates the local
+      file if the header is newer than the modified time (mtime) of the local
+      file.
+
+      _HTTP_ URIs can include a user information component so that Puppet can
+      retrieve file metadata and content from HTTP servers that require HTTP Basic
+      authentication. For example `https://<user>:<pass>@<server>:<port>/path/to/file.`
+
+      When connecting to _HTTPS_ servers, Puppet trusts CA certificates in the
+      puppet-agent certificate bundle and the Puppet CA. You can configure Puppet
+      to trust additional CA certificates using the `Puppet[:ssl_trust_store]`
+      setting.
 
       Multiple `source` values can be specified as an array, and Puppet will
       use the first source that exists. This can be used to serve different
@@ -104,8 +113,8 @@ module Puppet
           # Ruby 1.9.3 and earlier have a URI bug in URI
           # to_s returns an ASCII string despite UTF-8 fragments
           # since its escaped its safe to universally call encode
-          # URI.unescape always returns strings in the original encoding
-          URI.unescape(uri_string.encode(Encoding::UTF_8))
+          # Puppet::Util.uri_unescape always returns strings in the original encoding
+          Puppet::Util.uri_unescape(uri_string.encode(Encoding::UTF_8))
         else
           source
         end
@@ -129,18 +138,6 @@ module Puppet
       metadata && metadata.checksum
     end
 
-    # Look up (if necessary) and return local content.
-    def content
-      return @content if @content
-      raise Puppet::DevError, _("No source for content was stored with the metadata") unless metadata.source
-
-      tmp = Puppet::FileServing::Content.indirection.find(metadata.source, :environment => resource.catalog.environment_instance, :links => resource[:links])
-      unless tmp
-        self.fail "Could not find any content at %s" % metadata.source
-      end
-      @content = tmp.content
-    end
-
     # Copy the values from the source to the resource.  Yay.
     def copy_source_values
       devfail "Somehow got asked to copy source values without any metadata" unless metadata
@@ -273,63 +270,77 @@ module Puppet
       end
     end
 
-    def each_chunk_from
-      if Puppet[:default_file_terminus] == :file_server
-        yield content
+    def each_chunk_from(&block)
+      if Puppet[:default_file_terminus] == :file_server && scheme == 'puppet' && (uri.host.nil? || uri.host.empty?)
+        chunk_file_from_disk(metadata.full_path, &block)
       elsif local?
-        chunk_file_from_disk { |chunk| yield chunk }
+        chunk_file_from_disk(full_path, &block)
       else
-        chunk_file_from_source { |chunk| yield chunk }
+        chunk_file_from_source(&block)
       end
     end
 
-    def chunk_file_from_disk
-      File.open(full_path, "rb") do |src|
+    def chunk_file_from_disk(local_path)
+      File.open(local_path, "rb") do |src|
         while chunk = src.read(8192) #rubocop:disable Lint/AssignmentInCondition
           yield chunk
         end
       end
     end
 
-    def get_from_puppet_source(source_uri, content_uri, &block)
-      options = { :environment => resource.catalog.environment_instance }
-      if content_uri
-        options[:code_id] = resource.catalog.code_id
-        request = Puppet::Indirector::Request.new(:static_file_content, :find, content_uri, nil, options)
-      else
-        request = Puppet::Indirector::Request.new(:file_content, :find, source_uri, nil, options)
-      end
+    def get_from_content_uri_source(url, &block)
+      session = Puppet.lookup(:http_session)
+      api = session.route_to(:fileserver, url: url)
 
-      request.do_request(:fileserver) do |req|
-        ssl_context = Puppet.lookup(:ssl_context)
-        connection = Puppet::Network::HttpPool.connection(req.server, req.port, ssl_context: ssl_context)
-        connection.request_get(Puppet::Network::HTTP::API::IndirectedRoutes.request_to_uri(req), add_accept_encoding({"Accept" => BINARY_MIME_TYPES}), &block)
-      end
+      api.get_static_file_content(
+        path: Puppet::Util.uri_unescape(url.path),
+        environment: resource.catalog.environment_instance.to_s,
+        code_id: resource.catalog.code_id,
+        &block
+      )
     end
 
-    def get_from_http_source(source_uri, &block)
-      Puppet::Util::HttpProxy.request_with_redirects(URI(source_uri), :get, &block)
+    def get_from_source_uri_source(url, &block)
+      session = Puppet.lookup(:http_session)
+      api = session.route_to(:fileserver, url: url)
+
+      api.get_file_content(
+        path: Puppet::Util.uri_unescape(url.path),
+        environment: resource.catalog.environment_instance.to_s,
+        &block
+      )
     end
 
-    def get_from_source(&block)
-      source_uri = metadata.source
-      if source_uri =~ /^https?:/
-        get_from_http_source(source_uri, &block)
-      else
-        get_from_puppet_source(source_uri, metadata.content_uri, &block)
+    def get_from_http_source(url, &block)
+      client = Puppet.runtime[:http]
+      client.get(url, options: {include_system_store: true}) do |response|
+        raise Puppet::HTTP::ResponseError.new(response) unless response.success?
+
+        response.read_body(&block)
       end
     end
 
-    def chunk_file_from_source
-      get_from_source do |response|
-        case response.code
-        when /^2/;  uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } }
-        else
-          # Raise the http error if we didn't get a 'success' of some kind.
-          message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
-          raise Net::HTTPError.new(message, response)
-        end
+    def chunk_file_from_source(&block)
+      if uri.scheme =~ /^https?/
+        # Historically puppet has not encoded the http(s) source URL before parsing
+        # it, for example, if the path contains spaces, then it must be URL encoded
+        # as %20 in the manifest. Puppet behaves the same when retrieving file
+        # metadata via http(s), see Puppet::Indirector::FileMetadata::Http#find.
+        url = URI.parse(metadata.source)
+        get_from_http_source(url, &block)
+      elsif metadata.content_uri
+        content_url = URI.parse(Puppet::Util.uri_encode(metadata.content_uri))
+        get_from_content_uri_source(content_url, &block)
+      else
+        get_from_source_uri_source(uri, &block)
       end
+    rescue Puppet::HTTP::ResponseError => e
+      handle_response_error(e.response)
+    end
+
+    def handle_response_error(response)
+      message = "Error #{response.code} on SERVER: #{response.body.empty? ? response.reason : response.body}"
+      raise Net::HTTPError.new(message, response.nethttp)
     end
   end