puppet 6.13.0 → 6.14.0

data/lib/puppet/indirector/file_metadata/rest.rb

@@ -6,4 +6,54 @@ class Puppet::Indirector::FileMetadata::Rest < Puppet::Indirector::REST
   desc "Retrieve file metadata via a REST HTTP interface."
 
   use_srv_service(:fileserver)
+
+  def find(request)
+    return super unless use_http_client?
+
+    url = URI.parse(Puppet::Util.uri_encode(request.uri))
+    session = Puppet.lookup(:http_session)
+    api = session.route_to(:fileserver, url: url)
+
+    api.get_file_metadata(
+      path: URI.unescape(url.path),
+      environment: request.environment.to_s,
+      links: request.options[:links],
+      checksum_type: request.options[:checksum_type],
+      source_permissions: request.options[:source_permissions]
+    )
+  rescue Puppet::HTTP::ResponseError => e
+    if e.response.code == 404
+      return nil unless request.options[:fail_on_404]
+
+      _, body = parse_response(e.response.nethttp)
+      msg = _("Find %{uri} resulted in 404 with the message: %{body}") % { uri: elide(e.response.url.path, 100), body: body }
+      raise Puppet::Error, msg
+    else
+      raise convert_to_http_error(e.response.nethttp)
+    end
+  end
+
+  def search(request)
+    return super unless use_http_client?
+
+    url = URI.parse(Puppet::Util.uri_encode(request.uri))
+    session = Puppet.lookup(:http_session)
+    api = session.route_to(:fileserver, url: url)
+
+    api.get_file_metadatas(
+      path: URI.unescape(url.path),
+      environment: request.environment.to_s,
+      recurse: request.options[:recurse],
+      recurselimit: request.options[:recurselimit],
+      ignore: request.options[:ignore],
+      links: request.options[:links],
+      checksum_type: request.options[:checksum_type],
+      source_permissions: request.options[:source_permissions],
+    )
+  rescue Puppet::HTTP::ResponseError => e
+    # since it's search, return empty array instead of nil
+    return [] if e.response.code == 404
+
+    raise convert_to_http_error(e.response.nethttp)
+  end
 end

data/lib/puppet/indirector/node/rest.rb

@@ -4,4 +4,27 @@ require 'puppet/indirector/rest'
 class Puppet::Node::Rest < Puppet::Indirector::REST
   desc "Get a node via REST. Puppet agent uses this to allow the puppet master
     to override its environment."
+
+  def find(request)
+    return super unless use_http_client?
+
+    session = Puppet.lookup(:http_session)
+    api = session.route_to(:puppet)
+    api.get_node(
+      request.key,
+      environment: request.environment.to_s,
+      configured_environment: request.options[:configured_environment],
+      transaction_uuid: request.options[:transaction_uuid]
+    )
+  rescue Puppet::HTTP::ResponseError => e
+    if e.response.code == 404
+      return nil unless request.options[:fail_on_404]
+
+      _, body = parse_response(e.response.nethttp)
+      msg = _("Find %{uri} resulted in 404 with the message: %{body}") % { uri: elide(e.response.url.path, 100), body: body }
+      raise Puppet::Error, msg
+    else
+      raise convert_to_http_error(e.response.nethttp)
+    end
+  end
 end

data/lib/puppet/indirector/report/rest.rb

@@ -7,6 +7,25 @@ class Puppet::Transaction::Report::Rest < Puppet::Indirector::REST
   use_port_setting(:report_port)
   use_srv_service(:report)
 
+  def save(request)
+    return super unless use_http_client?
+
+    session = Puppet.lookup(:http_session)
+    api = session.route_to(:report)
+    response = api.put_report(
+      request.key,
+      request.instance,
+      environment: request.environment.to_s
+    )
+    content_type, body = parse_response(response)
+    deserialize_save(content_type, body)
+  rescue Puppet::HTTP::ResponseError => e
+    return nil if e.response.code == 404
+
+    raise convert_to_http_error(e.response.nethttp)
+  end
+
+  # This is called by the superclass when not using our httpclient.
   def handle_response(request, response)
     if !response.is_a?(Net::HTTPSuccess)
       server_version = response[Puppet::Network::HTTP::HEADER_PUPPET_VERSION]

data/lib/puppet/indirector/rest.rb

@@ -52,6 +52,12 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus
     Puppet::Util::Connection.determine_port(port_setting, server_setting)
   end
 
+  # Should we use puppet's http client to make requests. Will return
+  # false when running in puppetserver
+  def use_http_client?
+    Puppet::Network::HttpPool.http_client_class == Puppet::Network::HTTP::Connection
+  end
+
   # Provide appropriate headers.
   def headers
     # yaml is not allowed on the network

data/lib/puppet/indirector/status/rest.rb

@@ -6,4 +6,21 @@ class Puppet::Indirector::Status::Rest < Puppet::Indirector::REST
   desc "Get puppet master's status via REST. Useful because it tests the health
     of both the web server and the indirector."
 
+  def find(request)
+    return super unless use_http_client?
+
+    session = Puppet.lookup(:http_session)
+    api = session.route_to(:puppet)
+    api.get_status(request.key)
+  rescue Puppet::HTTP::ResponseError => e
+    if e.response.code == 404
+      return nil unless request.options[:fail_on_404]
+
+      _, body = parse_response(e.response.nethttp)
+      msg = _("Find %{uri} resulted in 404 with the message: %{body}") % { uri: elide(e.response.url.path, 100), body: body }
+      raise Puppet::Error, msg
+    else
+      raise convert_to_http_error(e.response.nethttp)
+    end
+  end
 end

data/lib/puppet/loaders.rb

@@ -1,3 +1,5 @@
+require 'puppet/concurrent/synchronized'
+
 module Puppet
   module Pops
     require 'puppet/pops/loaders'
@@ -22,6 +24,10 @@ module Puppet
       require 'puppet/pops/loader/predefined_loader'
       require 'puppet/pops/loader/generic_plan_instantiator'
       require 'puppet/pops/loader/puppet_plan_instantiator'
+
+      # The implementation of synchronized applies it to all subclasses so we
+      # want to add it to be base class after any subclasses are created
+      Loader.include Puppet::Concurrent::Synchronized
     end
   end
 

data/lib/puppet/network/http/base_pool.rb

@@ -4,7 +4,7 @@
 class Puppet::Network::HTTP::BasePool
   def start(site, verifier, http)
     Puppet.debug("Starting connection for #{site}")
-    if verifier
+    if site.use_ssl?
       verifier.setup_connection(http)
       begin
         http.start

data/lib/puppet/network/http/pool.rb

@@ -75,9 +75,14 @@ class Puppet::Network::HTTP::Pool < Puppet::Network::HTTP::BasePool
   # @api private
   def borrow(site, verifier)
     @pool[site] = active_sessions(site)
-    index = @pool[site].index { |session| verifier.reusable?(session.verifier) }
+    index = @pool[site].index do |session|
+      (verifier.nil? && session.verifier.nil?) ||
+        (!verifier.nil? && verifier.reusable?(session.verifier))
+    end
     session = index ? @pool[site].delete_at(index) : nil
     if session
+      @pool.delete(site) if @pool[site].empty?
+
       Puppet.debug("Using cached connection for #{site}")
       session.connection
     else

data/lib/puppet/provider/group/groupadd.rb

@@ -74,6 +74,14 @@ Puppet::Type.type(:group).provide :groupadd, :parent => Puppet::Provider::NameSe
     cmd
   end
 
+  def validate_members(members)
+    members.each do |member|
+      member.split(',').each do |user|
+        Etc.getpwnam(user.strip)
+      end
+    end
+  end
+
   def modifycmd(param, value)
     if @resource.forcelocal? || @resource[:members]
       cmd = [command(:localmodify)]
@@ -83,6 +91,7 @@ Puppet::Type.type(:group).provide :groupadd, :parent => Puppet::Provider::NameSe
     end
 
     if param == :members
+      validate_members(value)
       value = members_to_s(value)
       purge_members if @resource[:auth_membership] && !members.empty?
     end
@@ -118,10 +127,6 @@ Puppet::Type.type(:group).provide :groupadd, :parent => Puppet::Provider::NameSe
     localmodify('-m', members_to_s(members), @resource.name)
   end
 
-  def member_valid?(user)
-    !!Etc.getpwnam(user)
-  end
-
   private
 
   def findgroup(key, value)

data/lib/puppet/runtime.rb

@@ -5,7 +5,9 @@ class Puppet::Runtime
   include Singleton
 
   def initialize
-    @runtime_services = {}
+    @runtime_services = {
+      'http' => proc { Puppet::HTTP::Client.new }
+    }
   end
   private :initialize
 
@@ -23,4 +25,9 @@ class Puppet::Runtime
   def []=(name, impl)
     @runtime_services[name] = impl
   end
+
+  # for testing
+  def clear
+    initialize
+  end
 end

data/lib/puppet/settings.rb

@@ -30,6 +30,7 @@ class Puppet::Settings
   require 'puppet/settings/value_translator'
   require 'puppet/settings/environment_conf'
   require 'puppet/settings/server_list_setting'
+  require 'puppet/settings/http_extra_headers_setting'
   require 'puppet/settings/certificate_revocation_setting'
 
   # local reference for convenience
@@ -727,6 +728,7 @@ class Puppet::Settings
       :priority   => PrioritySetting,
       :autosign   => AutosignSetting,
       :server_list => ServerListSetting,
+      :http_extra_headers => HttpExtraHeadersSetting,
       :certificate_revocation => CertificateRevocationSetting
   }
 

data/lib/puppet/settings/http_extra_headers_setting.rb

@@ -0,0 +1,25 @@
+class Puppet::Settings::HttpExtraHeadersSetting < Puppet::Settings::BaseSetting
+
+  def type
+    :http_extra_headers
+  end
+
+  def munge(headers)
+    return headers if headers.is_a?(Hash)
+
+    headers = headers.split(/\s*,\s*/) if headers.is_a?(String)
+
+    raise ArgumentError, _("Expected an Array, String, or Hash, got a %{klass}") % { klass: headers.class } unless headers.is_a?(Array)
+
+    headers.map! { |header|
+      case header
+      when String
+        header.split(':')
+      when Array
+        header
+      else
+        raise ArgumentError, _("Expected an Array or String, got a %{klass}") % { klass: header.class }
+      end
+    }
+  end
+end

data/lib/puppet/ssl/state_machine.rb

@@ -278,6 +278,10 @@ class Puppet::SSL::StateMachine
       else
         Puppet.info(_("Will try again in %{time} seconds.") % {time: time})
 
+        # close persistent connections and session state before sleeping
+        Puppet.runtime['http'].close
+        @machine.session = nil
+
         Kernel.sleep(time)
 
         # our ssl directory may have been cleaned while we were

data/lib/puppet/test/test_helper.rb

@@ -136,10 +136,12 @@ module Puppet::Test
         {
           trusted_information:
             Puppet::Context::TrustedInformation.new('local', 'testing', {}, { "trusted_testhelper" => true }),
-          ssl_context: Puppet::SSL::SSLContext.new(cacerts: []).freeze
+          ssl_context: Puppet::SSL::SSLContext.new(cacerts: []).freeze,
+          http_session: proc { Puppet.runtime["http"].create_session }
         },
         "Context for specs")
 
+      Puppet.runtime.clear
       Puppet::Parser::Functions.reset
       Puppet::Application.clear!
       Puppet::Util::Profiler.clear

data/lib/puppet/type/file.rb

@@ -110,6 +110,19 @@ Puppet::Type.newtype(:file) do
       balancer to direct all filebucket traffic to a single master, or use
       something like an out-of-band rsync task to synchronize the content on all
       masters.
+
+      > **Note**: Enabling and using the backup option, and by extension the 
+        filebucket resource, requires appropriate planning and management to ensure 
+        that sufficient disk space is available for the file backups. Generally, you 
+        can implement this using one of the following two options:
+        - Use a `find` command and `crontab` entry to retain only the last X days 
+        of file backups. For example,
+        
+        ```shell script
+        find /opt/puppetlabs/server/data/puppetserver/bucket -type f -mtime +45 -atime +45 -print0 | xargs -0 rm
+        ```
+        
+        - Restrict the directory to a maximum size after which the oldest items are removed.
     EOT
 
     defaultto "puppet"

data/lib/puppet/type/file/source.rb

@@ -2,10 +2,7 @@ require 'puppet/file_serving/content'
 require 'puppet/file_serving/metadata'
 require 'puppet/file_serving/terminus_helper'
 
-require 'puppet/util/http_proxy'
-require 'puppet/network/http'
-require 'puppet/network/http/api/indirected_routes'
-require 'puppet/network/http/compression'
+require 'puppet/http'
 
 module Puppet
   # Copy files from a local or remote source.  This state *only* does any work
@@ -14,11 +11,6 @@ module Puppet
   # this state, during retrieval, modifies the appropriate other states
   # so that things get taken care of appropriately.
   Puppet::Type.type(:file).newparam(:source) do
-    include Puppet::Network::HTTP::Compression.module
-
-    BINARY_MIME_TYPES = [
-      Puppet::Network::FormatHandler.format_for('binary').mime
-    ].join(', ').freeze
 
     attr_accessor :source, :local
     desc <<-'EOT'
@@ -129,18 +121,6 @@ module Puppet
       metadata && metadata.checksum
     end
 
-    # Look up (if necessary) and return local content.
-    def content
-      return @content if @content
-      raise Puppet::DevError, _("No source for content was stored with the metadata") unless metadata.source
-
-      tmp = Puppet::FileServing::Content.indirection.find(metadata.source, :environment => resource.catalog.environment_instance, :links => resource[:links])
-      unless tmp
-        self.fail "Could not find any content at %s" % metadata.source
-      end
-      @content = tmp.content
-    end
-
     # Copy the values from the source to the resource.  Yay.
     def copy_source_values
       devfail "Somehow got asked to copy source values without any metadata" unless metadata
@@ -273,63 +253,72 @@ module Puppet
       end
     end
 
-    def each_chunk_from
-      if Puppet[:default_file_terminus] == :file_server
-        yield content
+    def each_chunk_from(&block)
+      if Puppet[:default_file_terminus] == :file_server && scheme == 'puppet' && (uri.host.nil? || uri.host.empty?)
+        chunk_file_from_disk(metadata.path, &block)
       elsif local?
-        chunk_file_from_disk { |chunk| yield chunk }
+        chunk_file_from_disk(full_path, &block)
       else
-        chunk_file_from_source { |chunk| yield chunk }
+        chunk_file_from_source(&block)
       end
     end
 
-    def chunk_file_from_disk
-      File.open(full_path, "rb") do |src|
+    def chunk_file_from_disk(local_path)
+      File.open(local_path, "rb") do |src|
         while chunk = src.read(8192) #rubocop:disable Lint/AssignmentInCondition
           yield chunk
         end
       end
     end
 
-    def get_from_puppet_source(source_uri, content_uri, &block)
-      options = { :environment => resource.catalog.environment_instance }
-      if content_uri
-        options[:code_id] = resource.catalog.code_id
-        request = Puppet::Indirector::Request.new(:static_file_content, :find, content_uri, nil, options)
-      else
-        request = Puppet::Indirector::Request.new(:file_content, :find, source_uri, nil, options)
-      end
+    def get_from_content_uri_source(url, &block)
+      session = Puppet.lookup(:http_session)
+      api = session.route_to(:fileserver, url: url)
 
-      request.do_request(:fileserver) do |req|
-        ssl_context = Puppet.lookup(:ssl_context)
-        connection = Puppet::Network::HttpPool.connection(req.server, req.port, ssl_context: ssl_context)
-        connection.request_get(Puppet::Network::HTTP::API::IndirectedRoutes.request_to_uri(req), add_accept_encoding({"Accept" => BINARY_MIME_TYPES}), &block)
-      end
+      api.get_static_file_content(
+        path: URI.unescape(url.path),
+        environment: resource.catalog.environment_instance.to_s,
+        code_id: resource.catalog.code_id,
+        &block
+      )
     end
 
-    def get_from_http_source(source_uri, &block)
-      Puppet::Util::HttpProxy.request_with_redirects(URI(source_uri), :get, &block)
+    def get_from_source_uri_source(url, &block)
+      session = Puppet.lookup(:http_session)
+      api = session.route_to(:fileserver, url: url)
+
+      api.get_file_content(
+        path: URI.unescape(url.path),
+        environment: resource.catalog.environment_instance.to_s,
+        &block
+      )
     end
 
-    def get_from_source(&block)
-      source_uri = metadata.source
-      if source_uri =~ /^https?:/
-        get_from_http_source(source_uri, &block)
-      else
-        get_from_puppet_source(source_uri, metadata.content_uri, &block)
+    def get_from_http_source(url, &block)
+      client = Puppet.runtime['http']
+      client.get(url) do |response|
+        raise Puppet::HTTP::ResponseError.new(response) unless response.success?
+
+        response.read_body(&block)
       end
     end
 
-    def chunk_file_from_source
-      get_from_source do |response|
-        case response.code
-        when /^2/;  uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } }
-        else
-          # Raise the http error if we didn't get a 'success' of some kind.
-          message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
-          raise Net::HTTPError.new(message, response)
-        end
+    def chunk_file_from_source(&block)
+      if uri.scheme =~ /^https?/
+        get_from_http_source(uri, &block)
+      elsif metadata.content_uri
+        content_url = URI.parse(Puppet::Util.uri_encode(metadata.content_uri))
+        get_from_content_uri_source(content_url, &block)
+      else
+        get_from_source_uri_source(uri, &block)
       end
+    rescue Puppet::HTTP::ResponseError => e
+      handle_response_error(e.response)
+    end
+
+    def handle_response_error(response)
+      message = "Error #{response.code} on SERVER: #{response.body.empty? ? response.reason : response.body}"
+      raise Net::HTTPError.new(message, response.nethttp)
     end
   end