puppet 6.12.0-universal-darwin → 6.17.0-universal-darwin

data/lib/puppet/forge.rb

@@ -254,6 +254,6 @@ class Puppet::Forge < SemanticPuppet::Dependency::Source
   def decode_uri(uri)
     return if uri.nil?
 
-    URI.decode(uri.tr('+', ' '))
+    Puppet::Util.uri_unescape(uri.tr('+', ' '))
   end
 end

data/lib/puppet/forge/cache.rb

@@ -25,7 +25,7 @@ class Puppet::Forge
         unless cached_file.file?
           if uri.scheme == 'file'
             # CGI.unescape butchers Uris that are escaped properly
-            FileUtils.cp(URI.unescape(uri.path), cached_file)
+            FileUtils.cp(Puppet::Util.uri_unescape(uri.path), cached_file)
           else
             # TODO: Handle HTTPS; probably should use repository.contact
             data = read_retrieve(uri)

data/lib/puppet/forge/repository.rb

@@ -37,18 +37,15 @@ class Puppet::Forge
         uri = URI(str)
 
         headers = { "User-Agent" => user_agent }
-        user = nil
-        password = nil
 
         if forge_authorization
+          uri.user = nil
+          uri.password = nil
           headers["Authorization"] = forge_authorization
-        elsif @uri.user && @uri.password
-          user = @uri.user
-          password = @uri.password
         end
 
-        http = Puppet.runtime['http']
-        response = http.get(uri, headers: headers, user: user, password: password, ssl_context: @ssl_context)
+        http = Puppet.runtime[:http]
+        response = http.get(uri, headers: headers, options: {ssl_context: @ssl_context})
         io.write(response.body) if io.respond_to?(:write)
         response
       rescue Puppet::SSL::CertVerifyError => e

data/lib/puppet/functions/call.rb

@@ -51,7 +51,7 @@
 #
 # Would notice the value of `$facts['processors']['count']` at the time when the `call` is made.
 #
-# * Deferred values supported since Puppet 5.6.0
+# * Deferred values supported since Puppet 6.0
 #
 # @since 5.0.0
 #

data/lib/puppet/functions/eyaml_lookup_key.rb

@@ -39,7 +39,7 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
       context.cache(nil, raw_data)
     end
     context.not_found unless raw_data.include?(key)
-    context.cache(key, decrypt_value(raw_data[key], context, options))
+    context.cache(key, decrypt_value(raw_data[key], context, options, key))
   end
 
   def load_data_hash(options, context)
@@ -62,22 +62,22 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
     end
   end
 
-  def decrypt_value(value, context, options)
+  def decrypt_value(value, context, options, key)
     case value
     when String
-      decrypt(value, context, options)
+      decrypt(value, context, options, key)
     when Hash
       result = {}
-      value.each_pair { |k, v| result[context.interpolate(k)] = decrypt_value(v, context, options) }
+      value.each_pair { |k, v| result[context.interpolate(k)] = decrypt_value(v, context, options, key) }
       result
     when Array
-      value.map { |v| decrypt_value(v, context, options) }
+      value.map { |v| decrypt_value(v, context, options, key) }
     else
       value
     end
   end
 
-  def decrypt(data, context, options)
+  def decrypt(data, context, options, key)
     if encrypted?(data)
       # Options must be set prior to each call to #parse since they end up as static variables in
       # the Options class. They cannot be set once before #decrypt_value is called, since each #decrypt
@@ -85,8 +85,13 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
       # config.
       #
       Hiera::Backend::Eyaml::Options.set(options)
-      tokens = Hiera::Backend::Eyaml::Parser::ParserFactory.hiera_backend_parser.parse(data)
-      data = tokens.map(&:to_plain_text).join.chomp
+      begin
+        tokens = Hiera::Backend::Eyaml::Parser::ParserFactory.hiera_backend_parser.parse(data)
+        data = tokens.map(&:to_plain_text).join.chomp
+      rescue StandardError => ex
+        raise Puppet::DataBinding::LookupError,
+          _("hiera-eyaml backend error decrypting %{data} when looking up %{key} in %{path}. Error was %{message}") % { data: data, key: key, path: options['path'], message: ex.message }
+      end
     end
     context.interpolate(data)
   end

data/lib/puppet/functions/filter.rb

@@ -38,6 +38,7 @@
 # $data = { "orange" => 0, "blueberry" => 1, "raspberry" => 2 }
 # $filtered_data = $data.filter |$items| { $items[0] =~ /berry$/ }
 # # $filtered_data = {blueberry => 1, raspberry => 2}
+# ```
 #
 # When the first argument is an array and the lambda has two parameters, Puppet passes the
 # array's indexes (enumerated from 0) in the first parameter and its values in the second

data/lib/puppet/functions/reduce.rb

@@ -39,11 +39,9 @@
 # values to the lambda.
 #
 # Puppet calls the lambda for each of the data structure's remaining values. For each
-# call, it passes the result of the previous call as the first parameter ($memo in the
+# call, it passes the result of the previous call as the first parameter (`$memo` in the
 # above examples) and the next value from the data structure as the second parameter
-# ($value).
-#
-# If the structure has one value, Puppet returns the value and does not call the lambda.
+# (`$value`).
 #
 # @example Using the `reduce` function
 #

data/lib/puppet/http.rb

@@ -11,13 +11,17 @@ module Puppet
     end
   end
 
+  # @api private
   module HTTP
     ACCEPT_ENCODING = "gzip;q=1.0,deflate;q=0.6,identity;q=0.3".freeze
+    HEADER_PUPPET_VERSION = "X-Puppet-Version".freeze
 
     require 'puppet/http/errors'
     require 'puppet/http/response'
     require 'puppet/http/service'
     require 'puppet/http/service/ca'
+    require 'puppet/http/service/compiler'
+    require 'puppet/http/service/file_server'
     require 'puppet/http/service/report'
     require 'puppet/http/session'
     require 'puppet/http/resolver'
@@ -27,5 +31,6 @@ module Puppet
     require 'puppet/http/client'
     require 'puppet/http/redirector'
     require 'puppet/http/retry_after_handler'
+    require 'puppet/http/external_client'
   end
 end

data/lib/puppet/http/client.rb

@@ -1,27 +1,85 @@
+#
+# @api private
+#
+# The client contains a pool of persistent HTTP connections and creates HTTP
+# sessions.
+#
 class Puppet::HTTP::Client
-  def initialize(pool: Puppet::Network::HTTP::Pool.new, ssl_context: nil, redirect_limit: 10, retry_limit: 100)
+
+  # @api private
+  # @return [Puppet::Network::HTTP::Pool] the pool instance associated with
+  #   this client
+  attr_reader :pool
+
+  #
+  # @api private
+  #
+  # Create a new http client instance. The client contains a pool of persistent
+  # HTTP connections and creates HTTP sessions.
+  #
+  # @param [Puppet::Network::HTTP::Pool] pool pool of persistent Net::HTTP
+  #   connections
+  # @param [Puppet::SSL::SSLContext] ssl_context ssl context to be used for
+  #   connections
+  # @param [Puppet::SSL::SSLContext] system_ssl_context the system ssl context
+  #   used if :include_system_store is set to true
+  # @param [Integer] redirect_limit default number of HTTP redirections to allow
+  #   in a given request. Can also be specified per-request.
+  # @param [Integer] retry_limit number of HTTP reties allowed in a given
+  #   request
+  #
+  def initialize(pool: Puppet::Network::HTTP::Pool.new(Puppet[:http_keepalive_timeout]), ssl_context: nil, system_ssl_context: nil, redirect_limit: 10, retry_limit: 100)
     @pool = pool
     @default_headers = {
       'X-Puppet-Version' => Puppet.version,
       'User-Agent' => Puppet[:http_user_agent],
     }.freeze
     @default_ssl_context = ssl_context
-    @redirector = Puppet::HTTP::Redirector.new(redirect_limit)
+    @default_system_ssl_context = system_ssl_context
+    @default_redirect_limit = redirect_limit
     @retry_after_handler = Puppet::HTTP::RetryAfterHandler.new(retry_limit, Puppet[:runinterval])
-    @resolvers = build_resolvers
   end
 
+  #
+  # @api private
+  #
+  # Create a new HTTP session. A session is the object through which services
+  # may be connected to and accessed.
+  #
+  # @return [Puppet::HTTP::Session] the newly created HTTP session
+  #
   def create_session
-    Puppet::HTTP::Session.new(self, @resolvers)
+    Puppet::HTTP::Session.new(self, build_resolvers)
   end
 
-  def connect(uri, ssl_context: nil, &block)
+  #
+  # @api private
+  #
+  # Open a connection to the given URI
+  #
+  # @param [URI] uri the connection destination
+  # @param [Hash] options
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  #
+  # @yield [Net::HTTP] If a block is given, yields an active http connection
+  #   from the pool
+  #
+  def connect(uri, options: {}, &block)
     start = Time.now
-    ctx = ssl_context ? ssl_context : default_ssl_context
-    site = Puppet::Network::HTTP::Site.from_uri(uri)
-    verifier = Puppet::SSL::Verifier.new(site.host, ctx)
+    verifier = nil
     connected = false
 
+    site = Puppet::Network::HTTP::Site.from_uri(uri)
+    if site.use_ssl?
+      ssl_context = options.fetch(:ssl_context, nil)
+      include_system_store = options.fetch(:include_system_store, false)
+      ctx = resolve_ssl_context(ssl_context, include_system_store)
+      verifier = Puppet::SSL::Verifier.new(site.host, ctx)
+    end
+
     @pool.with_connection(site, verifier) do |http|
       connected = true
       if block_given?
@@ -43,117 +101,228 @@ class Puppet::HTTP::Client
                 {uri: uri, elapsed: elapsed(start), message: e.message}, e, connected)
   end
 
-  def get(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil, &block)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a GET HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @yield [Puppet::HTTP::Response] if a block is given yields the response
+  #
+  # @return [String] if a block is not given, returns the response body
+  #
+  def get(url, headers: {}, params: {}, options: {}, &block)
+    url = encode_query(url, params)
 
     request = Net::HTTP::Get.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+    execute_streaming(request, options: options) do |response|
       if block_given?
         yield response
       else
-        response.read_body
+        response.body
       end
     end
   end
 
-  def head(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a HEAD HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def head(url, headers: {}, params: {}, options: {})
+    url = encode_query(url, params)
 
     request = Net::HTTP::Head.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    execute_streaming(request, options: options) do |response|
+      response.body
     end
   end
 
-  def put(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a PUT HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [String] body the body of the PUT request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [String] :content_type the type of the body content
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def put(url, body, headers: {}, params: {}, options: {})
+    raise ArgumentError, "'put' requires a string 'body' argument" unless body.is_a?(String)
+    url = encode_query(url, params)
 
     request = Net::HTTP::Put.new(url, @default_headers.merge(headers))
     request.body = body
-    request['Content-Length'] = body.bytesize
-    request['Content-Type'] = content_type
+    request.content_length = body.bytesize
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    raise ArgumentError, "'put' requires a 'content-type' header" unless request['Content-Type']
+
+    execute_streaming(request, options: options) do |response|
+      response.body
     end
   end
 
-  def post(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil, &block)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a POST HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [String] body the body of the POST request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options passed through to the request execution
+  # @option options [String] :content_type the type of the body content
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def post(url, body, headers: {}, params: {}, options: {}, &block)
+    raise ArgumentError, "'post' requires a string 'body' argument" unless body.is_a?(String)
+    url = encode_query(url, params)
 
     request = Net::HTTP::Post.new(url, @default_headers.merge(headers))
     request.body = body
-    request['Content-Length'] = body.bytesize
-    request['Content-Type'] = content_type
+    request.content_length = body.bytesize
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+    raise ArgumentError, "'post' requires a 'content-type' header" unless request['Content-Type']
+
+    execute_streaming(request, options: options) do |response|
       if block_given?
         yield response
       else
-        response.read_body
+        response.body
       end
     end
   end
 
-  def delete(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
-    query = encode_params(params)
-    unless query.empty?
-      url = url.dup
-      url.query = query
-    end
+  #
+  # @api private
+  #
+  # Submits a DELETE HTTP request to the given url
+  #
+  # @param [URI] url the location to submit the http request
+  # @param [Hash] headers merged with the default headers defined by the client
+  # @param [Hash] params encoded and set as the url query
+  # @param [Hash] options options hash passed through to the request execution
+  # @option options [Puppet::SSL::SSLContext] :ssl_context (nil) ssl context to
+  #   be used for connections
+  # @option options [Boolean] :include_system_store (false) if we should include
+  #   the system store for connection
+  # @param options [Integer] :redirect_limit number of HTTP redirections to allow
+  #   for this request.
+  #
+  # @return [String] the body of the request response
+  #
+  def delete(url, headers: {}, params: {}, options: {})
+    url = encode_query(url, params)
 
     request = Net::HTTP::Delete.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    execute_streaming(request, options: options) do |response|
+      response.body
     end
   end
 
+  #
+  # @api private
+  #
+  # Close persistent connections in the pool
+  #
   def close
     @pool.close
   end
 
+  protected
+
+  def encode_query(url, params)
+    return url if params.empty?
+
+    url = url.dup
+    url.query = encode_params(params)
+    url
+  end
+
   private
 
-  def execute_streaming(request, ssl_context:, user: nil, password: nil, &block)
+  def execute_streaming(request, options: {}, &block)
+    redirector = Puppet::HTTP::Redirector.new(options.fetch(:redirect_limit, @default_redirect_limit))
+
+    basic_auth = options.fetch(:basic_auth, nil)
+    unless basic_auth
+      if request.uri.user && request.uri.password
+        basic_auth = { user: request.uri.user, password: request.uri.password }
+      end
+    end
+
     redirects = 0
     retries = 0
+    response = nil
+    done = false
 
-    loop do
-      connect(request.uri, ssl_context: ssl_context) do |http|
-        apply_auth(request, user, password)
+    while !done do
+      connect(request.uri, options: options) do |http|
+        apply_auth(request, basic_auth)
 
+        # don't call return within the `request` block
         http.request(request) do |nethttp|
-          response = Puppet::HTTP::Response.new(nethttp)
+          response = Puppet::HTTP::Response.new(nethttp, request.uri)
           begin
             Puppet.debug("HTTP #{request.method.upcase} #{request.uri} returned #{response.code} #{response.reason}")
 
-            if @redirector.redirect?(request, response)
-              request = @redirector.redirect_to(request, response, redirects)
+            if redirector.redirect?(request, response)
+              request = redirector.redirect_to(request, response, redirects)
               redirects += 1
               next
             elsif @retry_after_handler.retry_after?(request, response)
               interval = @retry_after_handler.retry_after_interval(request, response, retries)
               retries += 1
               if interval
+                if http.started?
+                  Puppet.debug("Closing connection for #{Puppet::Network::HTTP::Site.from_uri(request.uri)}")
+                  http.finish
+                end
                 Puppet.warning(_("Sleeping for %{interval} seconds before retrying the request") % { interval: interval })
                 ::Kernel.sleep(interval)
                 next
@@ -165,13 +334,45 @@ class Puppet::HTTP::Client
             response.drain
           end
 
-          return response
+          done = true
         end
       end
     end
+
+    response
+  end
+
+  def expand_into_parameters(data)
+    data.inject([]) do |params, key_value|
+      key, value = key_value
+
+      expanded_value = case value
+                       when Array
+                         value.collect { |val| [key, val] }
+                       else
+                         [key_value]
+                       end
+
+      params.concat(expand_primitive_types_into_parameters(expanded_value))
+    end
+  end
+
+  def expand_primitive_types_into_parameters(data)
+    data.inject([]) do |params, key_value|
+      key, value = key_value
+      case value
+      when nil
+        params
+      when true, false, String, Symbol, Integer, Float
+        params << [key, value]
+      else
+        raise Puppet::HTTP::SerializationError, _("HTTP REST queries cannot handle values of type '%{klass}'") % { klass: value.class }
+      end
+    end
   end
 
   def encode_params(params)
+    params = expand_into_parameters(params)
     params.map do |key, value|
       "#{key}=#{Puppet::Util.uri_query_encode(value.to_s)}"
     end.join('&')
@@ -189,13 +390,30 @@ class Puppet::HTTP::Client
     end
   end
 
-  def default_ssl_context
-    @default_ssl_context || Puppet.lookup(:ssl_context)
+  def resolve_ssl_context(ssl_context, include_system_store)
+    if ssl_context
+      raise Puppet::HTTP::HTTPError, "The ssl_context and include_system_store parameters are mutually exclusive" if include_system_store
+      ssl_context
+    elsif include_system_store
+      system_ssl_context
+    else
+      @default_ssl_context || Puppet.lookup(:ssl_context)
+    end
+  end
+
+  def system_ssl_context
+    return @default_system_ssl_context if @default_system_ssl_context
+
+    cert_provider = Puppet::X509::CertProvider.new
+    cacerts = cert_provider.load_cacerts || []
+
+    ssl = Puppet::SSL::SSLProvider.new
+    @default_system_ssl_context = ssl.create_system_context(cacerts: cacerts)
   end
 
-  def apply_auth(request, user, password)
-    if user && password
-      request.basic_auth(user, password)
+  def apply_auth(request, basic_auth)
+    if basic_auth
+      request.basic_auth(basic_auth[:user], basic_auth[:password])
     end
   end
 
@@ -208,14 +426,16 @@ class Puppet::HTTP::Client
 
     server_list_setting = Puppet.settings.setting(:server_list)
     if server_list_setting.value && !server_list_setting.value.empty?
-      services = [:puppet]
-
-      # If we have not explicitly set :ca_server either on the command line or
-      # in puppet.conf, we want to be able to try the servers defined by
-      # :server_list when resolving the :ca service. Otherwise, :server_list
-      # should only be used with the :puppet service.
-      if !Puppet.settings.set_by_config?(:ca_server)
-        services << :ca
+      # use server list to resolve all services
+      services = Puppet::HTTP::Service::SERVICE_NAMES.dup
+
+      # except if it's been explicitly set
+      if Puppet.settings.set_by_config?(:ca_server)
+        services.delete(:ca)
+      end
+
+      if Puppet.settings.set_by_config?(:report_server)
+        services.delete(:report)
       end
 
       resolvers << Puppet::HTTP::Resolver::ServerList.new(self, server_list_setting: server_list_setting, default_port: Puppet[:masterport], services: services)