puppet 6.29.0 → 7.0.0

data/lib/puppet/http/client.rb

@@ -1,23 +1,96 @@
+# The HTTP client provides methods for making `GET`, `POST`, etc requests to
+# HTTP(S) servers. It also provides methods for resolving Puppetserver REST
+# service endpoints using SRV records and settings (such as `server_list`,
+# `server`, `ca_server`, etc). Once a service endpoint has been resolved, there
+# are methods for making REST requests (such as getting a node, sending facts,
+# etc).
 #
-# @api private
+# The client uses persistent HTTP connections by default unless the `Connection:
+# close` header is specified and supports streaming response bodies.
 #
-# The client contains a pool of persistent HTTP connections and creates HTTP
-# sessions.
+# By default the client only trusts the Puppet CA for HTTPS connections. However,
+# if the `include_system_store` request option is set to true, then Puppet will
+# trust certificates in the puppet-agent CA bundle.
 #
+# @example To access the HTTP client:
+#   client = Puppet.runtime[:http]
+#
+# @example To make an HTTP GET request:
+#   response = client.get(URI("http://www.example.com"))
+#
+# @example To make an HTTPS GET request, trusting the puppet CA and certs in Puppet's CA bundle:
+#   response = client.get(URI("https://www.example.com"), include_system_store: true)
+#
+# @example To use a URL containing special characters, such as spaces:
+#  response = client.get(URI(Puppet::Util.uri_encode("https://www.example.com/path to file")))
+#
+# @example To pass query parameters:
+#   response = client.get(URI("https://www.example.com"), query: {'q' => 'puppet'})
+#
+# @example To pass custom headers:
+#   response = client.get(URI("https://www.example.com"), headers: {'Accept-Content' => 'application/json'})
+#
+# @example To check if the response is successful (2xx):
+#   response = client.get(URI("http://www.example.com"))
+#   puts response.success?
+#
+# @example To get the response code and reason:
+#   response = client.get(URI("http://www.example.com"))
+#   unless response.success?
+#     puts "HTTP #{response.code} #{response.reason}"
+#    end
+#
+# @example To read response headers:
+#   response = client.get(URI("http://www.example.com"))
+#   puts response['Content-Type']
+#
+# @example To stream the response body:
+#   client.get(URI("http://www.example.com")) do |response|
+#     if response.success?
+#       response.read_body do |data|
+#         puts data
+#       end
+#     end
+#   end
+#
+# @example To handle exceptions:
+#   begin
+#     client.get(URI("https://www.example.com"))
+#   rescue Puppet::HTTP::ResponseError => e
+#     puts "HTTP #{e.response.code} #{e.response.reason}"
+#   rescue Puppet::HTTP::ConnectionError => e
+#     puts "Connection error #{e.message}"
+#   rescue Puppet::SSL::SSLError => e
+#     puts "SSL error #{e.message}"
+#   rescue Puppet::HTTP::HTTPError => e
+#     puts "General HTTP error #{e.message}"
+#   end
+#
+# @example To route to the `:puppet` service:
+#   session = client.create_session
+#   service = session.route_to(:puppet)
+#
+# @example To make a node request:
+#   node = service.get_node(Puppet[:certname], environment: 'production')
+#
+# @example To submit facts:
+#   facts = Puppet::Indirection::Facts.indirection.find(Puppet[:certname])
+#   service.put_facts(Puppet[:certname], environment: 'production', facts: facts)
+#
+# @example To submit a report to the `:report` service:
+#   report = Puppet::Transaction::Report.new
+#   service = session.route_to(:report)
+#   service.put_report(Puppet[:certname], report, environment: 'production')
+#
+# @api public
 class Puppet::HTTP::Client
 
-  # @api private
-  # @return [Puppet::Network::HTTP::Pool] the pool instance associated with
-  #   this client
   attr_reader :pool
 
+  # Create a new http client instance. Use `Puppet.runtime[:http]` to get
+  # the current client instead of creating an instance of this class.
   #
-  # @api private
-  #
-  # Create a new http client instance. The client contains a pool of persistent
-  # HTTP connections and creates HTTP sessions.
-  #
-  # @param [Puppet::Network::HTTP::Pool] pool pool of persistent Net::HTTP
+  # @param [Puppet::HTTP::Pool] pool pool of persistent Net::HTTP
   #   connections
   # @param [Puppet::SSL::SSLContext] ssl_context ssl context to be used for
   #   connections
@@ -25,10 +98,10 @@ class Puppet::HTTP::Client
   #   used if :include_system_store is set to true
   # @param [Integer] redirect_limit default number of HTTP redirections to allow
   #   in a given request. Can also be specified per-request.
-  # @param [Integer] retry_limit number of HTTP retries allowed in a given
+  # @param [Integer] retry_limit number of HTTP reties allowed in a given
   #   request
   #
-  def initialize(pool: Puppet::Network::HTTP::Pool.new(Puppet[:http_keepalive_timeout]), ssl_context: nil, system_ssl_context: nil, redirect_limit: 10, retry_limit: 100)
+  def initialize(pool: Puppet::HTTP::Pool.new(Puppet[:http_keepalive_timeout]), ssl_context: nil, system_ssl_context: nil, redirect_limit: 10, retry_limit: 100)
     @pool = pool
     @default_headers = {
       'X-Puppet-Version' => Puppet.version,
@@ -40,22 +113,19 @@ class Puppet::HTTP::Client
     @retry_after_handler = Puppet::HTTP::RetryAfterHandler.new(retry_limit, Puppet[:runinterval])
   end
 
-  #
-  # @api private
-  #
   # Create a new HTTP session. A session is the object through which services
   # may be connected to and accessed.
   #
   # @return [Puppet::HTTP::Session] the newly created HTTP session
   #
+  # @api public
   def create_session
     Puppet::HTTP::Session.new(self, build_resolvers)
   end
 
-  #
-  # @api private
-  #
-  # Open a connection to the given URI
+  # Open a connection to the given URI. It is typically not necessary to call
+  # this method as the client will create connections as needed when a request
+  # is made.
   #
   # @param [URI] uri the connection destination
   # @param [Hash] options
@@ -63,16 +133,12 @@ class Puppet::HTTP::Client
   #   be used for connections
   # @option options [Boolean] :include_system_store (false) if we should include
   #   the system store for connection
-  #
-  # @yield [Net::HTTP] If a block is given, yields an active http connection
-  #   from the pool
-  #
   def connect(uri, options: {}, &block)
     start = Time.now
     verifier = nil
     connected = false
 
-    site = Puppet::Network::HTTP::Site.from_uri(uri)
+    site = Puppet::HTTP::Site.from_uri(uri)
     if site.use_ssl?
       ssl_context = options.fetch(:ssl_context, nil)
       include_system_store = options.fetch(:include_system_store, false)
@@ -101,88 +167,73 @@ class Puppet::HTTP::Client
                 {uri: uri, elapsed: elapsed(start), message: e.message}, e, connected)
   end
 
-  #
-  # @api private
-  #
+  # These options apply to all HTTP request methods
+  #
+  # @!macro [new] request_options
+  #   @param [Hash] options HTTP request options. Options not recognized by the
+  #     HTTP implementation will be ignored.
+  #   @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #     be used for connections
+  #   @option options [Boolean] :include_system_store (false) if we should include
+  #     the system store for connection
+  #   @option options [Integer] :redirect_limit (10) The maximum number of HTTP
+  #     redirections to allow for this request.
+  #   @option options [Hash] :basic_auth A map of `:username` => `String` and
+  #     `:password` => `String`
+  #   @option options [String] :metric_id The metric id used to track metrics
+  #     on requests.
+
   # Submits a GET HTTP request to the given url
   #
   # @param [URI] url the location to submit the http request
   # @param [Hash] headers merged with the default headers defined by the client
   # @param [Hash] params encoded and set as the url query
-  # @param [Hash] options passed through to the request execution
-  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
-  #   be used for connections
-  # @option options [Boolean] :include_system_store (false) if we should include
-  #   the system store for connection
-  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
-  #   for this request.
+  # @!macro request_options
   #
   # @yield [Puppet::HTTP::Response] if a block is given yields the response
   #
-  # @return [String] if a block is not given, returns the response body
+  # @return [Puppet::HTTP::Response] the response
   #
+  # @api public
   def get(url, headers: {}, params: {}, options: {}, &block)
     url = encode_query(url, params)
 
     request = Net::HTTP::Get.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, options: options) do |response|
-      if block_given?
-        yield response
-      else
-        response.body
-      end
-    end
+    execute_streaming(request, options: options, &block)
   end
 
-  #
-  # @api private
-  #
   # Submits a HEAD HTTP request to the given url
   #
   # @param [URI] url the location to submit the http request
   # @param [Hash] headers merged with the default headers defined by the client
   # @param [Hash] params encoded and set as the url query
-  # @param [Hash] options passed through to the request execution
-  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
-  #   be used for connections
-  # @option options [Boolean] :include_system_store (false) if we should include
-  #   the system store for connection
-  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
-  #   for this request.
+  # @!macro request_options
   #
-  # @return [String] the body of the request response
+  # @return [Puppet::HTTP::Response] the response
   #
+  # @api public
   def head(url, headers: {}, params: {}, options: {})
     url = encode_query(url, params)
 
     request = Net::HTTP::Head.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, options: options) do |response|
-      response.body
-    end
+    execute_streaming(request, options: options)
   end
 
-  #
-  # @api private
-  #
   # Submits a PUT HTTP request to the given url
   #
   # @param [URI] url the location to submit the http request
   # @param [String] body the body of the PUT request
-  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] headers merged with the default headers defined by the client. The
+  #   `Content-Type` header is required and should correspond to the type of data passed
+  #   as the `body` argument.
   # @param [Hash] params encoded and set as the url query
-  # @param [Hash] options passed through to the request execution
-  # @option options [String] :content_type the type of the body content
-  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
-  #   be used for connections
-  # @option options [Boolean] :include_system_store (false) if we should include
-  #   the system store for connection
-  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
-  #   for this request.
+  # @!macro request_options
   #
-  # @return [String] the body of the request response
+  # @return [Puppet::HTTP::Response] the response
   #
+  # @api public
   def put(url, body, headers: {}, params: {}, options: {})
     raise ArgumentError, "'put' requires a string 'body' argument" unless body.is_a?(String)
     url = encode_query(url, params)
@@ -193,31 +244,24 @@ class Puppet::HTTP::Client
 
     raise ArgumentError, "'put' requires a 'content-type' header" unless request['Content-Type']
 
-    execute_streaming(request, options: options) do |response|
-      response.body
-    end
+    execute_streaming(request, options: options)
   end
 
-  #
-  # @api private
-  #
   # Submits a POST HTTP request to the given url
   #
   # @param [URI] url the location to submit the http request
   # @param [String] body the body of the POST request
-  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] headers merged with the default headers defined by the client. The
+  #   `Content-Type` header is required and should correspond to the type of data passed
+  #   as the `body` argument.
   # @param [Hash] params encoded and set as the url query
-  # @param [Hash] options passed through to the request execution
-  # @option options [String] :content_type the type of the body content
-  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
-  #   be used for connections
-  # @option options [Boolean] :include_system_store (false) if we should include
-  #   the system store for connection
-  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
-  #   for this request.
+  # @!macro request_options
+  #
+  # @yield [Puppet::HTTP::Response] if a block is given yields the response
   #
-  # @return [String] the body of the request response
+  # @return [Puppet::HTTP::Response] the response
   #
+  # @api public
   def post(url, body, headers: {}, params: {}, options: {}, &block)
     raise ArgumentError, "'post' requires a string 'body' argument" unless body.is_a?(String)
     url = encode_query(url, params)
@@ -228,68 +272,34 @@ class Puppet::HTTP::Client
 
     raise ArgumentError, "'post' requires a 'content-type' header" unless request['Content-Type']
 
-    execute_streaming(request, options: options) do |response|
-      if block_given?
-        yield response
-      else
-        response.body
-      end
-    end
+    execute_streaming(request, options: options, &block)
   end
 
-  #
-  # @api private
-  #
-  # Submits a DELETE HTTP request to the given url
+  # Submits a DELETE HTTP request to the given url.
   #
   # @param [URI] url the location to submit the http request
   # @param [Hash] headers merged with the default headers defined by the client
   # @param [Hash] params encoded and set as the url query
-  # @param [Hash] options options hash passed through to the request execution
-  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
-  #   be used for connections
-  # @option options [Boolean] :include_system_store (false) if we should include
-  #   the system store for connection
-  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
-  #   for this request.
+  # @!macro request_options
   #
-  # @return [String] the body of the request response
+  # @return [Puppet::HTTP::Response] the response
   #
+  # @api public
   def delete(url, headers: {}, params: {}, options: {})
     url = encode_query(url, params)
 
     request = Net::HTTP::Delete.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, options: options) do |response|
-      response.body
-    end
+    execute_streaming(request, options: options)
   end
 
+  # Close persistent connections in the pool.
   #
-  # @api private
-  #
-  # Close persistent connections in the pool
+  # @return [void]
   #
+  # @api public
   def close
     @pool.close
-    @default_ssl_context = nil
-    @default_system_ssl_context = nil
-  end
-
-  def default_ssl_context
-    cert = Puppet::X509::CertProvider.new
-    password = cert.load_private_key_password
-
-    ssl = Puppet::SSL::SSLProvider.new
-    ctx = ssl.load_context(certname: Puppet[:certname], password: password)
-    ssl.print(ctx)
-    ctx
-  rescue => e
-    # TRANSLATORS: `message` is an already translated string of why SSL failed to initialize
-    Puppet.log_exception(e, _("Failed to initialize SSL: %{message}") % { message: e.message })
-    # TRANSLATORS: `puppet agent -t` is a command and should not be translated
-    Puppet.err(_("Run `puppet agent -t`"))
-    raise e
   end
 
   protected
@@ -304,6 +314,21 @@ class Puppet::HTTP::Client
 
   private
 
+  # Connect or borrow a connection from the pool to the host and port associated
+  # with the request's URL. Then execute the HTTP request, retrying and
+  # following redirects as needed, and return the HTTP response. The response
+  # body will always be fully drained/consumed when this method returns.
+  #
+  # If a block is provided, then the response will be yielded to the caller,
+  # allowing the response body to be streamed.
+  #
+  # If the request/response did not result in an exception and the caller did
+  # not ask for the connection to be closed (via Connection: close), then the
+  # connection will be returned to the pool.
+  #
+  # @yieldparam [Puppet::HTTP::Response] response The final response, after
+  # following redirects and retrying
+  # @return [Puppet::HTTP::Response]
   def execute_streaming(request, options: {}, &block)
     redirector = Puppet::HTTP::Redirector.new(options.fetch(:redirect_limit, @default_redirect_limit))
 
@@ -321,11 +346,11 @@ class Puppet::HTTP::Client
 
     while !done do
       connect(request.uri, options: options) do |http|
-        apply_auth(request, basic_auth) if redirects.zero?
+        apply_auth(request, basic_auth)
 
         # don't call return within the `request` block
         http.request(request) do |nethttp|
-          response = Puppet::HTTP::Response.new(nethttp, request.uri)
+          response = Puppet::HTTP::ResponseNetHTTP.new(request.uri, nethttp)
           begin
             Puppet.debug("HTTP #{request.method.upcase} #{request.uri} returned #{response.code} #{response.reason}")
 
@@ -338,7 +363,7 @@ class Puppet::HTTP::Client
               retries += 1
               if interval
                 if http.started?
-                  Puppet.debug("Closing connection for #{Puppet::Network::HTTP::Site.from_uri(request.uri)}")
+                  Puppet.debug("Closing connection for #{Puppet::HTTP::Site.from_uri(request.uri)}")
                   http.finish
                 end
                 Puppet.warning(_("Sleeping for %{interval} seconds before retrying the request") % { interval: interval })
@@ -347,8 +372,15 @@ class Puppet::HTTP::Client
               end
             end
 
-            yield response
+            if block_given?
+              yield response
+            else
+              response.body
+            end
           ensure
+            # we need to make sure the response body is fully consumed before
+            # the connection is put back in the pool, otherwise the response
+            # for one request could leak into a future response.
             response.drain
           end
 
@@ -426,9 +458,7 @@ class Puppet::HTTP::Client
     cacerts = cert_provider.load_cacerts || []
 
     ssl = Puppet::SSL::SSLProvider.new
-    @default_system_ssl_context = ssl.create_system_context(cacerts: cacerts, include_client_cert: true)
-    ssl.print(@default_system_ssl_context)
-    @default_system_ssl_context
+    @default_system_ssl_context = ssl.create_system_context(cacerts: cacerts)
   end
 
   def apply_auth(request, basic_auth)

data/lib/puppet/{network/resolver.rb → http/dns.rb}

@@ -1,7 +1,7 @@
 require 'resolv'
 
-module Puppet::Network
-  class Resolver
+module Puppet::HTTP
+  class DNS
 
     class CacheEntry
       attr_reader :records, :ttl, :resolution_time

data/lib/puppet/http/errors.rb

@@ -1,14 +1,26 @@
 module Puppet::HTTP
+  # A base class for puppet http errors
+  # @api public
   class HTTPError < Puppet::Error; end
 
+  # A connection error such as if the server refuses the connection.
+  # @api public
   class ConnectionError < HTTPError; end
 
+  # A failure to route to the server such as if the `server_list` is exhausted.
+  # @api public
   class RouteError < HTTPError; end
 
+  # An HTTP protocol error, such as the server's response missing a required header.
+  # @api public
   class ProtocolError < HTTPError; end
 
+  # An error serializing or deserializing an object via REST.
+  # @api public
   class SerializationError < HTTPError; end
 
+  # An error due to an unsuccessful HTTP response, such as HTTP 500.
+  # @api public
   class ResponseError < HTTPError
     attr_reader :response
 
@@ -18,12 +30,16 @@ module Puppet::HTTP
     end
   end
 
+  # An error if asked to follow too many redirects (such as HTTP 301).
+  # @api public
   class TooManyRedirects < HTTPError
     def initialize(addr)
       super(_("Too many HTTP redirections for %{addr}") % { addr: addr})
     end
   end
 
+  # An error if asked to retry (such as HTTP 503) too many times.
+  # @api public
   class TooManyRetryAfters < HTTPError
     def initialize(addr)
       super(_("Too many HTTP retries for %{addr}") % { addr: addr})

data/lib/puppet/http/external_client.rb

@@ -1,4 +1,3 @@
-#
 # Adapts an external http_client_class to the HTTP client API. The former
 # is typically registered by puppetserver and only implements a subset of
 # the Puppet::Network::HTTP::Connection methods. As a result, only the
@@ -7,10 +6,10 @@
 #
 # @api private
 class Puppet::HTTP::ExternalClient < Puppet::HTTP::Client
-  # Create an external http client
+
+  # Create an external http client.
   #
   # @param [Class] http_client_class The class to create to handle the request
-  # @api private
   def initialize(http_client_class)
     @http_client_class = http_client_class
   end
@@ -23,7 +22,7 @@ class Puppet::HTTP::ExternalClient < Puppet::HTTP::Client
     options[:use_ssl] = url.scheme == 'https'
 
     client = @http_client_class.new(url.host, url.port, options)
-    response = Puppet::HTTP::Response.new(client.get(url.request_uri, headers, options), url)
+    response = Puppet::HTTP::ResponseNetHTTP.new(url, client.get(url.request_uri, headers, options))
 
     if block_given?
       yield response
@@ -45,7 +44,7 @@ class Puppet::HTTP::ExternalClient < Puppet::HTTP::Client
     options[:use_ssl] = url.scheme == 'https'
 
     client = @http_client_class.new(url.host, url.port, options)
-    response = Puppet::HTTP::Response.new(client.post(url.request_uri, body, headers, options), url)
+    response = Puppet::HTTP::ResponseNetHTTP.new(url, client.post(url.request_uri, body, headers, options))
 
     if block_given?
       yield response
@@ -58,8 +57,7 @@ class Puppet::HTTP::ExternalClient < Puppet::HTTP::Client
     raise Puppet::HTTP::HTTPError.new(e.message, e)
   end
 
-  # Close the external http client.
-  #
+  # (see Puppet::HTTP::Client#close)
   # @api private
   def close
     # This is a noop as puppetserver doesn't provide a way to close its http client.

data/lib/puppet/{network/http → http}/factory.rb

@@ -1,15 +1,14 @@
 require 'puppet/ssl/openssl_loader'
 require 'net/http'
-require 'puppet/util/http_proxy'
+require 'puppet/http'
 
-# Factory for <tt>Net::HTTP</tt> objects.
+# Factory for `Net::HTTP` objects.
 #
-# Encapsulates the logic for creating a <tt>Net::HTTP</tt> object based on the
-# specified {Puppet::Network::HTTP::Site Site} and puppet settings.
+# Encapsulates the logic for creating a `Net::HTTP` object based on the
+# specified {Site} and puppet settings.
 #
 # @api private
-#
-class Puppet::Network::HTTP::Factory
+class Puppet::HTTP::Factory
   @@openssl_initialized = false
 
   KEEP_ALIVE_TIMEOUT = 2**31 - 1
@@ -25,20 +24,14 @@ class Puppet::Network::HTTP::Factory
   def create_connection(site)
     Puppet.debug("Creating new connection for #{site}")
 
-    http = Puppet::Util::HttpProxy.proxy(URI(site.addr))
+    http = Puppet::HTTP::Proxy.proxy(URI(site.addr))
     http.use_ssl = site.use_ssl?
-    if site.use_ssl?
-      http.min_version = OpenSSL::SSL::TLS1_VERSION if http.respond_to?(:min_version)
-      http.ciphers = Puppet[:ciphers]
-    end
     http.read_timeout = Puppet[:http_read_timeout]
     http.open_timeout = Puppet[:http_connect_timeout]
     http.keep_alive_timeout = KEEP_ALIVE_TIMEOUT if http.respond_to?(:keep_alive_timeout=)
 
-    if http.respond_to?(:max_retries)
-      # 0 means make one request and never retry
-      http.max_retries = 0
-    end
+    # 0 means make one request and never retry
+    http.max_retries = 0
 
     if Puppet[:sourceaddress]
       Puppet.debug("Using source IP #{Puppet[:sourceaddress]}")