puppet 6.13.0-universal-darwin → 6.18.0-universal-darwin

data/lib/puppet/functions/filter.rb

@@ -38,6 +38,7 @@
 # $data = { "orange" => 0, "blueberry" => 1, "raspberry" => 2 }
 # $filtered_data = $data.filter |$items| { $items[0] =~ /berry$/ }
 # # $filtered_data = {blueberry => 1, raspberry => 2}
+# ```
 #
 # When the first argument is an array and the lambda has two parameters, Puppet passes the
 # array's indexes (enumerated from 0) in the first parameter and its values in the second

data/lib/puppet/functions/lstrip.rb

@@ -12,15 +12,15 @@
 #
 # @example Removing leading space from a String
 # ```puppet
-# "\n\thello".lstrip()
-# camelcase("\n\thello")
+# "\n\thello ".lstrip()
+# lstrip("\n\thello ")
 # ```
 # Would both result in `"hello"`
 #
 # @example Removing leading space from strings in an Array
 # ```puppet
-# ["\n\thello", "\n\thi"].lstrip()
-# lstrip(["\n\thello", "\n\thi"])
+# ["\n\thello ", "\n\thi "].lstrip()
+# lstrip(["\n\thello ", "\n\thi "])
 # ```
 # Would both result in `['hello', 'hi']`
 #

data/lib/puppet/functions/reduce.rb

@@ -39,11 +39,9 @@
 # values to the lambda.
 #
 # Puppet calls the lambda for each of the data structure's remaining values. For each
-# call, it passes the result of the previous call as the first parameter ($memo in the
+# call, it passes the result of the previous call as the first parameter (`$memo` in the
 # above examples) and the next value from the data structure as the second parameter
-# ($value).
-#
-# If the structure has one value, Puppet returns the value and does not call the lambda.
+# (`$value`).
 #
 # @example Using the `reduce` function
 #

data/lib/puppet/functions/reverse_each.rb

@@ -84,7 +84,7 @@ Puppet::Functions.create_function(:reverse_each) do
 
   def reverse_each(iterable)
     # produces an Iterable
-    Puppet::Pops::Types::Iterable.asserted_iterable(self, iterable).reverse_each
+    Puppet::Pops::Types::Iterable.asserted_iterable(self, iterable, true).reverse_each
   end
 
   def reverse_each_block(iterable, &block)

data/lib/puppet/functions/rstrip.rb

@@ -12,15 +12,15 @@
 #
 # @example Removing trailing space from a String
 # ```puppet
-# "hello\n\t".lstrip()
-# camelcase("hello\n\t")
+# " hello\n\t".rstrip()
+# rstrip(" hello\n\t")
 # ```
 # Would both result in `"hello"`
 #
 # @example Removing trailing space from strings in an Array
 # ```puppet
-# ["hello\n\t", "hi\n\t"].lstrip()
-# lstrip(["hello\n\t", "hi\n\t"])
+# [" hello\n\t", " hi\n\t"].rstrip()
+# rstrip([" hello\n\t", " hi\n\t"])
 # ```
 # Would both result in `['hello', 'hi']`
 #

data/lib/puppet/functions/step.rb

@@ -88,7 +88,7 @@ Puppet::Functions.create_function(:step) do
 
   def step(iterable, step)
     # produces an Iterable
-    Puppet::Pops::Types::Iterable.asserted_iterable(self, iterable).step(step)
+    Puppet::Pops::Types::Iterable.asserted_iterable(self, iterable, true).step(step)
   end
 
   def step_block(iterable, step, &block)

data/lib/puppet/functions/strip.rb

@@ -12,15 +12,15 @@
 #
 # @example Removing leading and trailing space from a String
 # ```puppet
-# " hello\n\t".lstrip()
-# camelcase(" hello\n\t")
+# " hello\n\t".strip()
+# strip(" hello\n\t")
 # ```
 # Would both result in `"hello"`
 #
 # @example Removing trailing space from strings in an Array
 # ```puppet
-# [" hello\n\t", " hi\n\t"].lstrip()
-# lstrip([" hello\n\t", " hi\n\t"])
+# [" hello\n\t", " hi\n\t"].strip()
+# strip([" hello\n\t", " hi\n\t"])
 # ```
 # Would both result in `['hello', 'hi']`
 #

data/lib/puppet/gettext/config.rb

@@ -65,7 +65,7 @@ module Puppet::GettextConfig
     return if @gettext_disabled || !gettext_loaded?
     domain_name = domain_name.to_sym
 
-    Puppet.debug "Reset text domain to #{domain_name.inspect}"
+    Puppet.debug { "Reset text domain to #{domain_name.inspect}" }
     FastGettext.add_text_domain(domain_name,
                                 type: :chain,
                                 chain: [],
@@ -115,10 +115,10 @@ module Puppet::GettextConfig
     domain_name = domain_name.to_sym
 
     if FastGettext.translation_repositories.include?(domain_name)
-      Puppet.debug "Use text domain #{domain_name.inspect}"
+      Puppet.debug { "Use text domain #{domain_name.inspect}" }
       FastGettext.text_domain = domain_name
     else
-      Puppet.debug "Requested unknown text domain #{domain_name.inspect}"
+      Puppet.debug { "Requested unknown text domain #{domain_name.inspect}" }
     end
   end
 
@@ -139,10 +139,10 @@ module Puppet::GettextConfig
 
     deleted = FastGettext.translation_repositories.delete(domain_name)
     if FastGettext.text_domain == domain_name
-      Puppet.debug "Deleted current text domain #{domain_name.inspect}: #{!deleted.nil?}"
+      Puppet.debug { "Deleted current text domain #{domain_name.inspect}: #{!deleted.nil?}" }
       FastGettext.text_domain = nil
     else
-      Puppet.debug "Deleted text domain #{domain_name.inspect}: #{!deleted.nil?}"
+      Puppet.debug { "Deleted text domain #{domain_name.inspect}: #{!deleted.nil?}" }
     end
   end
 

data/lib/puppet/gettext/module_translations.rb

@@ -13,9 +13,9 @@ module Puppet::ModuleTranslations
 
       module_name = mod.forge_name.tr('/', '-')
       if Puppet::GettextConfig.load_translations(module_name, mod.locale_directory, :po)
-        Puppet.debug "Loaded translations for #{module_name}."
+        Puppet.debug { "Loaded translations for #{module_name}." }
       elsif Puppet::GettextConfig.gettext_loaded?
-        Puppet.debug "Could not find translation files for #{module_name} at #{mod.locale_directory}. Skipping translation initialization."
+        Puppet.debug { "Could not find translation files for #{module_name} at #{mod.locale_directory}. Skipping translation initialization." }
       else
         Puppet.warn_once("gettext_unavailable", "gettext_unavailable", "No gettext library found, skipping translation initialization.")
       end
@@ -31,9 +31,9 @@ module Puppet::ModuleTranslations
     Dir.glob("#{vardir}/locales/#{locale}/*.po") do |f|
       module_name = File.basename(f, ".po")
       if Puppet::GettextConfig.load_translations(module_name, File.join(vardir, "locales"), :po)
-        Puppet.debug "Loaded translations for #{module_name}."
+        Puppet.debug { "Loaded translations for #{module_name}." }
       elsif Puppet::GettextConfig.gettext_loaded?
-        Puppet.debug "Could not load translations for #{module_name}."
+        Puppet.debug { "Could not load translations for #{module_name}." }
       else
         Puppet.warn_once("gettext_unavailable", "gettext_unavailable", "No gettext library found, skipping translation initialization.")
       end

data/lib/puppet/http.rb

@@ -11,8 +11,10 @@ module Puppet
     end
   end
 
+  # @api private
   module HTTP
     ACCEPT_ENCODING = "gzip;q=1.0,deflate;q=0.6,identity;q=0.3".freeze
+    HEADER_PUPPET_VERSION = "X-Puppet-Version".freeze
 
     require 'puppet/http/errors'
     require 'puppet/http/response'
@@ -29,5 +31,6 @@ module Puppet
     require 'puppet/http/client'
     require 'puppet/http/redirector'
     require 'puppet/http/retry_after_handler'
+    require 'puppet/http/external_client'
   end
 end

data/lib/puppet/http/client.rb

@@ -1,27 +1,85 @@
+#
+# @api private
+#
+# The client contains a pool of persistent HTTP connections and creates HTTP
+# sessions.
+#
 class Puppet::HTTP::Client
-  def initialize(pool: Puppet::Network::HTTP::Pool.new, ssl_context: nil, redirect_limit: 10, retry_limit: 100)
+
+  # @api private
+  # @return [Puppet::Network::HTTP::Pool] the pool instance associated with
+  #   this client
+  attr_reader :pool
+
+  #
+  # @api private
+  #
+  # Create a new http client instance. The client contains a pool of persistent
+  # HTTP connections and creates HTTP sessions.
+  #
+  # @param [Puppet::Network::HTTP::Pool] pool pool of persistent Net::HTTP
+  #   connections
+  # @param [Puppet::SSL::SSLContext] ssl_context ssl context to be used for
+  #   connections
+  # @param [Puppet::SSL::SSLContext] system_ssl_context the system ssl context
+  #   used if :include_system_store is set to true
+  # @param [Integer] redirect_limit default number of HTTP redirections to allow
+  #   in a given request. Can also be specified per-request.
+  # @param [Integer] retry_limit number of HTTP reties allowed in a given
+  #   request
+  #
+  def initialize(pool: Puppet::Network::HTTP::Pool.new(Puppet[:http_keepalive_timeout]), ssl_context: nil, system_ssl_context: nil, redirect_limit: 10, retry_limit: 100)
     @pool = pool
     @default_headers = {
       'X-Puppet-Version' => Puppet.version,
       'User-Agent' => Puppet[:http_user_agent],
     }.freeze
     @default_ssl_context = ssl_context
-    @redirector = Puppet::HTTP::Redirector.new(redirect_limit)
+    @default_system_ssl_context = system_ssl_context
+    @default_redirect_limit = redirect_limit
     @retry_after_handler = Puppet::HTTP::RetryAfterHandler.new(retry_limit, Puppet[:runinterval])
-    @resolvers = build_resolvers
   end
 
+  #
+  # @api private
+  #
+  # Create a new HTTP session. A session is the object through which services
+  # may be connected to and accessed.
+  #
+  # @return [Puppet::HTTP::Session] the newly created HTTP session
+  #
   def create_session
-    Puppet::HTTP::Session.new(self, @resolvers)
+    Puppet::HTTP::Session.new(self, build_resolvers)
   end
 
-  def connect(uri, ssl_context: nil, &block)
+  #
+  # @api private
+  #
+  # Open a connection to the given URI
+  #
+  # @param [URI] uri the connection destination
+  # @param [Hash] options
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  #
+  # @yield [Net::HTTP] If a block is given, yields an active http connection
+  #   from the pool
+  #
+  def connect(uri, options: {}, &block)
     start = Time.now
-    ctx = ssl_context ? ssl_context : default_ssl_context
-    site = Puppet::Network::HTTP::Site.from_uri(uri)
-    verifier = Puppet::SSL::Verifier.new(site.host, ctx)
+    verifier = nil
     connected = false
 
+    site = Puppet::Network::HTTP::Site.from_uri(uri)
+    if site.use_ssl?
+      ssl_context = options.fetch(:ssl_context, nil)
+      include_system_store = options.fetch(:include_system_store, false)
+      ctx = resolve_ssl_context(ssl_context, include_system_store)
+      verifier = Puppet::SSL::Verifier.new(site.host, ctx)
+    end
+
     @pool.with_connection(site, verifier) do |http|
       connected = true
       if block_given?
@@ -43,117 +101,228 @@ class Puppet::HTTP::Client
                 {uri: uri, elapsed: elapsed(start), message: e.message}, e, connected)
   end
 
-  def get(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil, &block)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a GET HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @yield [Puppet::HTTP::Response] if a block is given yields the response
+  #
+  # @return [String] if a block is not given, returns the response body
+  #
+  def get(url, headers: {}, params: {}, options: {}, &block)
+    url = encode_query(url, params)
 
     request = Net::HTTP::Get.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+    execute_streaming(request, options: options) do |response|
       if block_given?
         yield response
       else
-        response.read_body
+        response.body
       end
     end
   end
 
-  def head(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a HEAD HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def head(url, headers: {}, params: {}, options: {})
+    url = encode_query(url, params)
 
     request = Net::HTTP::Head.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    execute_streaming(request, options: options) do |response|
+      response.body
     end
   end
 
-  def put(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a PUT HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [String] body the body of the PUT request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [String] :content_type the type of the body content
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def put(url, body, headers: {}, params: {}, options: {})
+    raise ArgumentError, "'put' requires a string 'body' argument" unless body.is_a?(String)
+    url = encode_query(url, params)
 
     request = Net::HTTP::Put.new(url, @default_headers.merge(headers))
     request.body = body
-    request['Content-Length'] = body.bytesize
-    request['Content-Type'] = content_type
+    request.content_length = body.bytesize
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    raise ArgumentError, "'put' requires a 'content-type' header" unless request['Content-Type']
+
+    execute_streaming(request, options: options) do |response|
+      response.body
     end
   end
 
-  def post(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil, &block)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a POST HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [String] body the body of the POST request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [String] :content_type the type of the body content
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def post(url, body, headers: {}, params: {}, options: {}, &block)
+    raise ArgumentError, "'post' requires a string 'body' argument" unless body.is_a?(String)
+    url = encode_query(url, params)
 
     request = Net::HTTP::Post.new(url, @default_headers.merge(headers))
     request.body = body
-    request['Content-Length'] = body.bytesize
-    request['Content-Type'] = content_type
+    request.content_length = body.bytesize
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+    raise ArgumentError, "'post' requires a 'content-type' header" unless request['Content-Type']
+
+    execute_streaming(request, options: options) do |response|
       if block_given?
         yield response
       else
-        response.read_body
+        response.body
       end
     end
   end
 
-  def delete(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a DELETE HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options options hash passed through to the request execution
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def delete(url, headers: {}, params: {}, options: {})
+    url = encode_query(url, params)
 
     request = Net::HTTP::Delete.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    execute_streaming(request, options: options) do |response|
+      response.body
     end
   end
 
+  #
+  # @api private
+  #
+  # Close persistent connections in the pool
+  #
   def close
     @pool.close
   end
 
+  protected
+
+  def encode_query(url, params)
+    return url if params.empty?
+
+    url = url.dup
+    url.query = encode_params(params)
+    url
+  end
+
   private
 
-  def execute_streaming(request, ssl_context:, user: nil, password: nil, &block)
+  def execute_streaming(request, options: {}, &block)
+    redirector = Puppet::HTTP::Redirector.new(options.fetch(:redirect_limit, @default_redirect_limit))
+
+    basic_auth = options.fetch(:basic_auth, nil)
+    unless basic_auth
+      if request.uri.user && request.uri.password
+        basic_auth = { user: request.uri.user, password: request.uri.password }
+      end
+    end
+
     redirects = 0
     retries = 0
+    response = nil
+    done = false
 
-    loop do
-      connect(request.uri, ssl_context: ssl_context) do |http|
-        apply_auth(request, user, password)
+    while !done do
+      connect(request.uri, options: options) do |http|
+        apply_auth(request, basic_auth)
 
+        # don't call return within the `request` block
         http.request(request) do |nethttp|
-          response = Puppet::HTTP::Response.new(nethttp)
+          response = Puppet::HTTP::Response.new(nethttp, request.uri)
           begin
             Puppet.debug("HTTP #{request.method.upcase} #{request.uri} returned #{response.code} #{response.reason}")
 
-            if @redirector.redirect?(request, response)
-              request = @redirector.redirect_to(request, response, redirects)
+            if redirector.redirect?(request, response)
+              request = redirector.redirect_to(request, response, redirects)
               redirects += 1
               next
             elsif @retry_after_handler.retry_after?(request, response)
               interval = @retry_after_handler.retry_after_interval(request, response, retries)
               retries += 1
               if interval
+                if http.started?
+                  Puppet.debug("Closing connection for #{Puppet::Network::HTTP::Site.from_uri(request.uri)}")
+                  http.finish
+                end
                 Puppet.warning(_("Sleeping for %{interval} seconds before retrying the request") % { interval: interval })
                 ::Kernel.sleep(interval)
                 next
@@ -165,10 +334,12 @@ class Puppet::HTTP::Client
             response.drain
           end
 
-          return response
+          done = true
         end
       end
     end
+
+    response
   end
 
   def expand_into_parameters(data)
@@ -219,13 +390,30 @@ class Puppet::HTTP::Client
     end
   end
 
-  def default_ssl_context
-    @default_ssl_context || Puppet.lookup(:ssl_context)
+  def resolve_ssl_context(ssl_context, include_system_store)
+    if ssl_context
+      raise Puppet::HTTP::HTTPError, "The ssl_context and include_system_store parameters are mutually exclusive" if include_system_store
+      ssl_context
+    elsif include_system_store
+      system_ssl_context
+    else
+      @default_ssl_context || Puppet.lookup(:ssl_context)
+    end
   end
 
-  def apply_auth(request, user, password)
-    if user && password
-      request.basic_auth(user, password)
+  def system_ssl_context
+    return @default_system_ssl_context if @default_system_ssl_context
+
+    cert_provider = Puppet::X509::CertProvider.new
+    cacerts = cert_provider.load_cacerts || []
+
+    ssl = Puppet::SSL::SSLProvider.new
+    @default_system_ssl_context = ssl.create_system_context(cacerts: cacerts)
+  end
+
+  def apply_auth(request, basic_auth)
+    if basic_auth
+      request.basic_auth(basic_auth[:user], basic_auth[:password])
     end
   end
 
@@ -238,14 +426,16 @@ class Puppet::HTTP::Client
 
     server_list_setting = Puppet.settings.setting(:server_list)
     if server_list_setting.value && !server_list_setting.value.empty?
-      services = [:puppet]
-
-      # If we have not explicitly set :ca_server either on the command line or
-      # in puppet.conf, we want to be able to try the servers defined by
-      # :server_list when resolving the :ca service. Otherwise, :server_list
-      # should only be used with the :puppet service.
-      if !Puppet.settings.set_by_config?(:ca_server)
-        services << :ca
+      # use server list to resolve all services
+      services = Puppet::HTTP::Service::SERVICE_NAMES.dup
+
+      # except if it's been explicitly set
+      if Puppet.settings.set_by_config?(:ca_server)
+        services.delete(:ca)
+      end
+
+      if Puppet.settings.set_by_config?(:report_server)
+        services.delete(:report)
       end
 
       resolvers << Puppet::HTTP::Resolver::ServerList.new(self, server_list_setting: server_list_setting, default_port: Puppet[:masterport], services: services)