puppet 5.0.0 → 6.0.0

data/lib/puppet/network/http/connection.rb

@@ -1,9 +1,10 @@
 require 'net/https'
 require 'puppet/ssl/host'
-require 'puppet/ssl/configuration'
 require 'puppet/ssl/validator'
 require 'puppet/network/http'
 require 'uri'
+require 'date'
+require 'time'
 
 module Puppet::Network::HTTP
 
@@ -75,7 +76,7 @@ module Puppet::Network::HTTP
     # @!macro common_options
     # @api public
     def get(path, headers = {}, options = {})
-      request_with_redirects(Net::HTTP::Get.new(path, headers), options)
+      do_request(Net::HTTP::Get.new(path, headers), options)
     end
 
     # @param path [String]
@@ -86,7 +87,7 @@ module Puppet::Network::HTTP
     def post(path, data, headers = nil, options = {})
       request = Net::HTTP::Post.new(path, headers)
       request.body = data
-      request_with_redirects(request, options)
+      do_request(request, options)
     end
 
     # @param path [String]
@@ -94,7 +95,7 @@ module Puppet::Network::HTTP
     # @!macro common_options
     # @api public
     def head(path, headers = {}, options = {})
-      request_with_redirects(Net::HTTP::Head.new(path, headers), options)
+      do_request(Net::HTTP::Head.new(path, headers), options)
     end
 
     # @param path [String]
@@ -102,7 +103,7 @@ module Puppet::Network::HTTP
     # @!macro common_options
     # @api public
     def delete(path, headers = {'Depth' => 'Infinity'}, options = {})
-      request_with_redirects(Net::HTTP::Delete.new(path, headers), options)
+      do_request(Net::HTTP::Delete.new(path, headers), options)
     end
 
     # @param path [String]
@@ -113,7 +114,7 @@ module Puppet::Network::HTTP
     def put(path, data, headers = nil, options = {})
       request = Net::HTTP::Put.new(path, headers)
       request.body = data
-      request_with_redirects(request, options)
+      do_request(request, options)
     end
 
     def request(method, *args)
@@ -164,7 +165,7 @@ module Puppet::Network::HTTP
 
     private
 
-    def request_with_redirects(request, options)
+    def do_request(request, options)
       current_request = request
       current_site = @site
       response = nil
@@ -177,9 +178,9 @@ module Puppet::Network::HTTP
 
           current_response = execute_request(connection, current_request)
 
-          if [301, 302, 307].include?(current_response.code.to_i)
-
-            # handle the redirection
+          case current_response.code.to_i
+          when 301, 302, 307
+            # handle redirection
             location = URI.parse(current_response['location'])
             current_site = current_site.move_to(location)
 
@@ -189,6 +190,8 @@ module Puppet::Network::HTTP
             request.each do |header, value|
               current_request[header] = value
             end
+          when 429, 503
+            response = handle_retry_after(current_response)
           else
             response = current_response
           end
@@ -200,6 +203,90 @@ module Puppet::Network::HTTP
       raise RedirectionLimitExceededException, _("Too many HTTP redirections for %{host}:%{port}") % { host: @host, port: @port }
     end
 
+    # Handles the Retry-After header of a HTTPResponse
+    #
+    # This method checks the response for a Retry-After header and handles
+    # it by sleeping for the indicated number of seconds. The response is
+    # returned unmodified if no Retry-After header is present.
+    #
+    # @param response [Net::HTTPResponse] A response received from the
+    #   HTTP client.
+    #
+    # @return [nil] Sleeps and returns nil if the response contained a
+    #   Retry-After header that indicated the request should be retried.
+    # @return [Net::HTTPResponse] Returns the `response` unmodified if
+    #   no Retry-After header was present or the Retry-After header could
+    #   not be parsed as an integer or RFC 2822 date.
+    def handle_retry_after(response)
+      retry_after = response['Retry-After']
+      return response if retry_after.nil?
+
+      retry_sleep = parse_retry_after_header(retry_after)
+      # Recover remote hostname if Net::HTTPResponse was generated by a
+      # method that fills in the uri attribute.
+      #
+      server_hostname = if response.uri.is_a?(URI)
+                          response.uri.host
+                        else
+                          # TRANSLATORS: Used in the phrase:
+                          # "Received a response from the remote server."
+                          _('the remote server')
+                        end
+
+      if retry_sleep.nil?
+        Puppet.err(_('Received a %{status_code} response from %{server_hostname}, but the Retry-After header value of "%{retry_after}" could not be converted to an integer or RFC 2822 date.') %
+                   {status_code: response.code,
+                    server_hostname: server_hostname,
+                    retry_after: retry_after.inspect})
+
+        return response
+      end
+
+      # Cap maximum sleep at the run interval of the Puppet agent.
+      retry_sleep = [retry_sleep, Puppet[:runinterval]].min
+
+      Puppet.warning(_('Received a %{status_code} response from %{server_hostname}. Sleeping for %{retry_sleep} seconds before retrying the request.') %
+                     {status_code: response.code,
+                      server_hostname: server_hostname,
+                      retry_sleep: retry_sleep})
+
+      ::Kernel.sleep(retry_sleep)
+
+      return nil
+    end
+
+    # Parse the value of a Retry-After header
+    #
+    # Parses a string containing an Integer or RFC 2822 datestamp and returns
+    # an integer number of seconds before a request can be retried.
+    #
+    # @param header_value [String] The value of the Retry-After header.
+    #
+    # @return [Integer] Number of seconds to wait before retrying the
+    #   request. Will be equal to 0 for the case of date that has already
+    #   passed.
+    # @return [nil] Returns `nil` when the `header_value` can't be
+    #   parsed as an Integer or RFC 2822 date.
+    def parse_retry_after_header(header_value)
+      retry_after = begin
+                      Integer(header_value)
+                    rescue TypeError, ArgumentError
+                      begin
+                        DateTime.rfc2822(header_value)
+                      rescue ArgumentError
+                        return nil
+                      end
+                    end
+
+      case retry_after
+      when Integer
+        retry_after
+      when DateTime
+        sleep = (retry_after.to_time - DateTime.now.to_time).to_i
+        (sleep > 0) ? sleep : 0
+      end
+    end
+
     def apply_options_to(request, options)
       request["User-Agent"] = Puppet[:http_user_agent]
 
@@ -209,7 +296,14 @@ module Puppet::Network::HTTP
     end
 
     def execute_request(connection, request)
+      start = Time.now
       connection.request(request)
+    rescue EOFError => e
+      elapsed = (Time.now - start).to_f.round(3)
+      uri = @site.addr + request.path.split('?')[0]
+      eof = EOFError.new(_('request %{uri} interrupted after %{elapsed} seconds') % {uri: uri, elapsed: elapsed})
+      eof.set_backtrace(e.backtrace) unless e.backtrace.empty?
+      raise eof
     end
 
     def with_connection(site, &block)
@@ -219,21 +313,7 @@ module Puppet::Network::HTTP
       end
       response
     rescue OpenSSL::SSL::SSLError => error
-      # can be nil
-      peer_cert = @verify.peer_certs.last
-
-      if peer_cert && !OpenSSL::SSL.verify_certificate_identity(peer_cert.content, site.host)
-        valid_certnames = [peer_cert.name, *peer_cert.subject_alt_names].uniq
-        msg = valid_certnames.length > 1 ? _("one of %{certnames}") % { certnames: valid_certnames.join(', ') } : valid_certnames.first
-        msg += _("Server hostname '%{host}' did not match server certificate; expected %{msg}") % { host: site.host, msg: msg }
-        raise Puppet::Error, msg, error.backtrace
-      elsif error.message.include? "certificate verify failed"
-        msg = error.message
-        msg << ": [" + @verify.verify_errors.join('; ') + "]"
-        raise Puppet::Error, msg, error.backtrace
-      else
-        raise
-      end
+      Puppet::Util::SSL.handle_connection_error(error, @verify, site.host)
     end
   end
 end

data/lib/puppet/network/http/error.rb

@@ -1,4 +1,4 @@
-require 'json'
+require 'puppet/util/json'
 
 module Puppet::Network::HTTP::Error
   Issues = Puppet::Network::HTTP::Issues
@@ -13,49 +13,49 @@ module Puppet::Network::HTTP::Error
     end
 
     def to_json
-      JSON({:message => message, :issue_kind => @issue_kind})
+      Puppet::Util::Json.dump({:message => message, :issue_kind => @issue_kind})
     end
   end
 
   class HTTPNotAcceptableError < HTTPError
     CODE = 406
     def initialize(message, issue_kind = Issues::RUNTIME_ERROR)
-      super(_("Not Acceptable: ") + message, CODE, issue_kind)
+      super(_("Not Acceptable: %{message}") % { message: message }, CODE, issue_kind)
     end
   end
 
   class HTTPNotFoundError < HTTPError
     CODE = 404
     def initialize(message, issue_kind = Issues::RUNTIME_ERROR)
-      super(_("Not Found: ") + message, CODE, issue_kind)
+      super(_("Not Found: %{message}") % { message: message }, CODE, issue_kind)
     end
   end
 
   class HTTPNotAuthorizedError < HTTPError
     CODE = 403
     def initialize(message, issue_kind = Issues::RUNTIME_ERROR)
-      super(_("Not Authorized: ") + message, CODE, issue_kind)
+      super(_("Not Authorized: %{message}") % { message: message }, CODE, issue_kind)
     end
   end
 
   class HTTPBadRequestError < HTTPError
     CODE = 400
     def initialize(message, issue_kind = Issues::RUNTIME_ERROR)
-      super(_("Bad Request: ") + message, CODE, issue_kind)
+      super(_("Bad Request: %{message}") % { message: message }, CODE, issue_kind)
     end
   end
 
   class HTTPMethodNotAllowedError < HTTPError
     CODE = 405
     def initialize(message, issue_kind = Issues::RUNTIME_ERROR)
-      super(_("Method Not Allowed: ") + message, CODE, issue_kind)
+      super(_("Method Not Allowed: %{message}") % { message: message }, CODE, issue_kind)
     end
   end
 
   class HTTPUnsupportedMediaTypeError < HTTPError
     CODE = 415
     def initialize(message, issue_kind = Issues::RUNTIME_ERROR)
-      super(_("Unsupported Media Type: ") + message, CODE, issue_kind)
+      super(_("Unsupported Media Type: %{message}") % { message: message }, CODE, issue_kind)
     end
   end
 
@@ -63,11 +63,11 @@ module Puppet::Network::HTTP::Error
     CODE = 500
 
     def initialize(original_error, issue_kind = Issues::RUNTIME_ERROR)
-      super(_("Server Error: ") + original_error.message, CODE, issue_kind)
+      super(_("Server Error: %{message}") % { message: original_error.message }, CODE, issue_kind)
     end
 
     def to_json
-      JSON({:message => message, :issue_kind => @issue_kind})
+      Puppet::Util::Json.dump({:message => message, :issue_kind => @issue_kind})
     end
   end
 end

data/lib/puppet/network/http/factory.rb

@@ -12,6 +12,8 @@ require 'puppet/util/http_proxy'
 class Puppet::Network::HTTP::Factory
   @@openssl_initialized = false
 
+  KEEP_ALIVE_TIMEOUT = 2**31 - 1
+
   def initialize
     # PUP-1411, make sure that openssl is initialized before we try to connect
     if ! @@openssl_initialized
@@ -37,6 +39,12 @@ class Puppet::Network::HTTP::Factory
     http.use_ssl = site.use_ssl?
     http.read_timeout = Puppet[:http_read_timeout]
     http.open_timeout = Puppet[:http_connect_timeout]
+    http.keep_alive_timeout = KEEP_ALIVE_TIMEOUT if http.respond_to?(:keep_alive_timeout=)
+
+    if Puppet[:sourceaddress]
+      Puppet.debug("Using source IP #{Puppet[:sourceaddress]}")
+      http.local_host = Puppet[:sourceaddress]
+    end
 
     if Puppet[:http_debug]
       http.set_debug_output($stderr)

data/lib/puppet/network/http/handler.rb

@@ -44,42 +44,78 @@ module Puppet::Network::HTTP::Handler
     format.is_a?(Puppet::Network::Format) ? format.mime : format
   end
 
-  # handle an HTTP request
-  def process(request, response)
-    new_response = Puppet::Network::HTTP::Response.new(self, response)
-
-    request_headers = headers(request)
-    request_params = params(request)
-    request_method = http_method(request)
+  # Create a generic puppet request from the implementation-specific request
+  # created by the web server
+  def make_generic_request(request)
     request_path = path(request)
 
-    new_request = Puppet::Network::HTTP::Request.new(request_headers, request_params, request_method, request_path, request_path, client_cert(request), body(request))
+    Puppet::Network::HTTP::Request.new(
+      headers(request),
+      params(request),
+      http_method(request),
+      request_path, # path
+      request_path, # routing_path
+      client_cert(request),
+      body(request)
+    )
+  end
+
+  def with_request_profiling(request)
+    profiler = configure_profiler(request.headers, request.params)
+
+    Puppet::Util::Profiler.profile(
+      _("Processed request %{request_method} %{request_path}") % { request_method: request.method, request_path: request.path },
+      [:http, request.method, request.path]
+    ) do
+      yield
+    end
+  ensure
+    remove_profiler(profiler) if profiler
+  end
 
-    response[Puppet::Network::HTTP::HEADER_PUPPET_VERSION] = Puppet.version
+  # handle an HTTP request
+  def process(external_request, response)
+    # The response_wrapper stores the response and modifies it as a side effect.
+    # The caller will use the original response
+    response_wrapper = Puppet::Network::HTTP::Response.new(self, response)
+    request = make_generic_request(external_request)
 
-    profiler = configure_profiler(request_headers, request_params)
+    set_puppet_version_header(response)
 
-    Puppet::Util::Profiler.profile(_("Processed request %{request_method} %{request_path}") % { request_method: request_method, request_path: request_path }, [:http, request_method, request_path]) do
-      if route = @routes.find { |r| r.matches?(new_request) }
-        route.process(new_request, new_response)
-      else
-        raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new(_("No route for %{request} %{path}") % { request: new_request.method, path: new_request.path }, HANDLER_NOT_FOUND)
+    respond_to_errors(response_wrapper) do
+      with_request_profiling(request) do
+        find_route_or_raise(request).process(request, response_wrapper)
       end
     end
+  end
 
+  def respond_to_errors(response)
+    yield
   rescue Puppet::Network::HTTP::Error::HTTPError => e
     Puppet.info(e.message)
-    new_response.respond_with(e.status, "application/json", e.to_json)
+    respond_with_http_error(response, e)
   rescue StandardError => e
     http_e = Puppet::Network::HTTP::Error::HTTPServerError.new(e)
-    log_msg = [http_e.message, *e.backtrace].join("\n")
-    Puppet.err(log_msg)
-    new_response.respond_with(http_e.status, "application/json", http_e.to_json)
-  ensure
-    if profiler
-      remove_profiler(profiler)
+    Puppet.err([http_e.message, *e.backtrace].join("\n"))
+    respond_with_http_error(response, http_e)
+  end
+
+  def respond_with_http_error(response, exception)
+    response.respond_with(exception.status, "application/json", exception.to_json)
+  end
+
+  def find_route_or_raise(request)
+    if route = @routes.find { |r| r.matches?(request) }
+      return route
+    else
+      raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new(
+              _("No route for %{request} %{path}") % { request: request.method, path: request.path },
+              HANDLER_NOT_FOUND)
     end
-    cleanup(request)
+  end
+
+  def set_puppet_version_header(response)
+    response[Puppet::Network::HTTP::HEADER_PUPPET_VERSION] = Puppet.version
   end
 
   # Set the response up, with the body and status.
@@ -131,10 +167,6 @@ module Puppet::Network::HTTP::Handler
     raise NotImplementedError
   end
 
-  def cleanup(request)
-    # By default, there is nothing to cleanup.
-  end
-
   def decode_params(params)
     params.select { |key, _| allowed_parameter?(key) }.inject({}) do |result, ary|
       param, value = ary

data/lib/puppet/network/http.rb

@@ -13,8 +13,6 @@ module Puppet::Network::HTTP
   require 'puppet/network/http/error'
   require 'puppet/network/http/route'
   require 'puppet/network/http/api'
-  require 'puppet/network/http/api/ca'
-  require 'puppet/network/http/api/ca/v1'
   require 'puppet/network/http/api/master'
   require 'puppet/network/http/api/master/v3'
   require 'puppet/network/http/handler'

data/lib/puppet/network/resolver.rb

@@ -1,86 +1,158 @@
 require 'resolv'
-module Puppet::Network; end
-
-module Puppet::Network::Resolver
-  # Iterate through the list of servers that service this hostname
-  # and yield each server/port since SRV records have ports in them
-  # It will override whatever masterport setting is already set.
-  def self.each_srv_record(domain, service_name = :puppet, &block)
-    if (domain.nil? or domain.empty?)
-      Puppet.debug "Domain not known; skipping SRV lookup"
-      return
+
+module Puppet::Network
+  class Resolver
+
+    class CacheEntry
+      attr_reader :records, :ttl, :resolution_time
+
+      def initialize(records)
+        @records = records
+        @resolution_time = Time.now
+        @ttl = choose_lowest_ttl(records)
+      end
+
+      def choose_lowest_ttl(records)
+        ttl = records.first.ttl
+        records.each do |rec|
+          if rec.ttl < ttl
+            ttl = rec.ttl
+          end
+        end
+        ttl
+      end
     end
 
-    Puppet.debug "Searching for SRV records for domain: #{domain}"
+    def initialize
+      @resolver = Resolv::DNS.new
 
-    case service_name
-      when :puppet then service = '_x-puppet'
-      when :ca     then service = '_x-puppet-ca'
-      when :report then service = '_x-puppet-report'
-      when :file   then service = '_x-puppet-fileserver'
-      else              service = "_x-puppet-#{service_name.to_s}"
+      # Stores DNS records per service, along with their TTL
+      # and the time at which they were resolved, for cache
+      # eviction.
+      @record_cache = {}
     end
-    srv_record = "#{service}._tcp.#{domain}"
-
-    resolver = Resolv::DNS.new
-    records = resolver.getresources(srv_record, Resolv::DNS::Resource::IN::SRV)
-    Puppet.debug "Found #{records.size} SRV records for: #{srv_record}"
-
-    if records.size == 0 && service_name != :puppet
-      # Try the generic :puppet service if no SRV records were found
-      # for the specific service.
-      each_srv_record(domain, :puppet, &block)
-    else
-      each_priority(records) do |priority, recs|
-        while next_rr = recs.delete(find_weighted_server(recs))
-          Puppet.debug "Yielding next server of #{next_rr.target.to_s}:#{next_rr.port}"
-          yield next_rr.target.to_s, next_rr.port
+
+    # Iterate through the list of records for this service
+    # and yield each server and port pair. Records are only fetched
+    # via DNS query the first time and cached for the duration of their
+    # service's TTL thereafter.
+    # @param [String] domain the domain to search for
+    # @param [Symbol] service_name the key of the service we are querying
+    # @yields [String, Integer] server and port of selected record
+    def each_srv_record(domain, service_name = :puppet, &block)
+      if (domain.nil? or domain.empty?)
+        Puppet.debug "Domain not known; skipping SRV lookup"
+        return
+      end
+
+      Puppet.debug "Searching for SRV records for domain: #{domain}"
+
+      case service_name
+        when :puppet then service = '_x-puppet'
+        when :file   then service = '_x-puppet-fileserver'
+        else              service = "_x-puppet-#{service_name.to_s}"
+      end
+      record_name = "#{service}._tcp.#{domain}"
+
+      if @record_cache.has_key?(service_name) && !expired?(service_name)
+        records = @record_cache[service_name].records
+        Puppet.debug "Using cached record for #{record_name}"
+      else
+        records = @resolver.getresources(record_name, Resolv::DNS::Resource::IN::SRV)
+        if records.size > 0
+          @record_cache[service_name] = CacheEntry.new(records)
+        end
+        Puppet.debug "Found #{records.size} SRV records for: #{record_name}"
+      end
+
+      if records.size == 0 && service_name != :puppet
+        # Try the generic :puppet service if no SRV records were found
+        # for the specific service.
+        each_srv_record(domain, :puppet, &block)
+      else
+        each_priority(records) do |recs|
+          while next_rr = recs.delete(find_weighted_server(recs))
+            Puppet.debug "Yielding next server of #{next_rr.target.to_s}:#{next_rr.port}"
+            yield next_rr.target.to_s, next_rr.port
+          end
         end
       end
     end
-  end
 
-  private
+    # Given a list of records of the same priority, chooses a random one
+    # from among them, favoring those with higher weights.
+    # @param [[Resolv::DNS::Resource::IN::SRV]] records a list of records
+    #        of the same priority
+    # @return [Resolv::DNS::Resource::IN:SRV] the chosen record
+    def find_weighted_server(records)
+      return nil if records.nil? || records.empty?
+      return records.first if records.size == 1
 
-  def self.each_priority(records)
-    pri_hash = records.inject({}) do |groups, element|
-      groups[element.priority] ||= []
-      groups[element.priority] << element
-      groups
+      # Calculate the sum of all weights in the list of resource records,
+      # This is used to then select hosts until the weight exceeds what
+      # random number we selected.  For example, if we have weights of 1 8 and 3:
+      #
+      # |-|--------|---|
+      #        ^
+      # We generate a random number 5, and iterate through the records, adding
+      # the current record's weight to the accumulator until the weight of the
+      # current record plus previous records is greater than the random number.
+      total_weight = records.inject(0) { |sum,record|
+        sum + weight(record)
+      }
+      current_weight = 0
+      chosen_weight  = 1 + Kernel.rand(total_weight)
+
+      records.each do |record|
+        current_weight += weight(record)
+        return record if current_weight >= chosen_weight
+      end
     end
 
-    pri_hash.keys.sort.each do |key|
-      yield key, pri_hash[key]
+    def weight(record)
+      record.weight == 0 ? 1 : record.weight * 10
     end
-  end
 
-  def self.find_weighted_server(records)
-    return nil if records.nil? || records.empty?
-    return records.first if records.size == 1
+    # Returns TTL for the cached records for this service.
+    # @param [String] service_name the service whose TTL we want
+    # @return [Integer] the TTL for this service, in seconds
+    def ttl(service_name)
+      return @record_cache[service_name].ttl
+    end
 
-    # Calculate the sum of all weights in the list of resource records,
-    # This is used to then select hosts until the weight exceeds what
-    # random number we selected.  For example, if we have weights of 1 8 and 3:
-    #
-    # |-|---|--------|
-    #        ^
-    # We generate a random number 5, and iterate through the records, adding
-    # the current record's weight to the accumulator until the weight of the
-    # current record plus previous records is greater than the random number.
-
-    total_weight = records.inject(0) { |sum,record|
-      sum + weight(record)
-    }
-    current_weight = 0
-    chosen_weight  = 1 + Kernel.rand(total_weight)
-
-    records.each do |record|
-      current_weight += weight(record)
-      return record if current_weight >= chosen_weight
+    # Checks if the cached entry for the given service has expired.
+    # @param [String] service_name the name of the service to check
+    # @return [Boolean] true if the entry has expired, false otherwise.
+    #                  Always returns true if the record had no TTL.
+    def expired?(service_name)
+      if entry = @record_cache[service_name]
+        return Time.now > (entry.resolution_time + entry.ttl)
+      else
+        return true
+      end
     end
-  end
 
-  def self.weight(record)
-    record.weight == 0 ? 1 : record.weight * 10
+    private
+
+    # Groups the records by their priority and yields the groups
+    # in order of highest to lowest priority (lowest to highest numbers),
+    # one at a time.
+    # { 1 => [records], 2 => [records], etc. }
+    #
+    # @param [[Resolv::DNS::Resource::IN::SRV]] records the list of
+    #        records for a given service
+    # @yields [[Resolv::DNS::Resource::IN::SRV]] a group of records of
+    #         the same priority
+    def each_priority(records)
+      pri_hash = records.inject({}) do |groups, element|
+        groups[element.priority] ||= []
+        groups[element.priority] << element
+        groups
+      end
+
+      pri_hash.keys.sort.each do |key|
+        yield pri_hash[key]
+      end
+    end
   end
 end

data/lib/puppet/network/rights.rb

@@ -34,7 +34,7 @@ class Rights
 
   def is_forbidden_and_why?(name, args = {})
     res = :nomatch
-    right = @rights.find do |acl|
+    @rights.find do |acl|
       found = false
       # an acl can return :dunno, which means "I'm not qualified to answer your question,
       # please ask someone else". This is used when for instance an acl matches, but not for the