puppet 6.13.0 → 6.14.0

data/lib/puppet/configurer/plugin_handler.rb

@@ -29,8 +29,17 @@ class Puppet::Configurer::PluginHandler
     result += plugin_fact_downloader.evaluate
     result += plugin_downloader.evaluate
 
+    # until file metadata/content are using the rest client, we need to check
+    # both :server_agent_version and the session to see if the server supports
+    # the "locales" mount
     server_agent_version = Puppet.lookup(:server_agent_version) { "0.0" }
-    if Gem::Version.new(server_agent_version) >= SUPPORTED_LOCALES_MOUNT_AGENT_VERSION
+    locales = Gem::Version.new(server_agent_version) >= SUPPORTED_LOCALES_MOUNT_AGENT_VERSION
+    unless locales
+      session = Puppet.lookup(:http_session)
+      locales = session.supports?(:puppet, 'locales')
+    end
+
+    if locales
       locales_downloader = Puppet::Configurer::Downloader.new(
         "locales",
         Puppet[:localedest],

data/lib/puppet/defaults.rb

@@ -1167,7 +1167,7 @@ EOT
         the request.
 
         For info on autosign configuration files, see
-        [the guide to Puppet's config files](https://puppet.com/docs/puppet/latest/config_about_settings.html).",
+        [the guide to Puppet's config files](https://puppet.com/docs/puppet/latest/config_file_autosign.html).",
     },
     :allow_duplicate_certs => {
       :default    => false,
@@ -1633,6 +1633,12 @@ EOT
       :default    => lambda { Puppet::Settings.domain_fact },
       :desc       => "The domain which will be queried to find the SRV records of servers to use.",
     },
+    :http_extra_headers => {
+      :default => [],
+      :type => :http_extra_headers,
+      :desc => "The list of extra headers that will be sent with http requests to the master.
+      The header definition consists of a name and a value separated by a colon." 
+    },
     :ignoreschedules => {
       :default    => false,
       :type       => :boolean,

data/lib/puppet/face/plugin.rb

@@ -41,7 +41,7 @@ Puppet::Face.define(:plugin, '0.0.1') do
     when_invoked do |options|
       remote_environment_for_plugins = Puppet::Node::Environment.remote(Puppet[:environment])
 
-      pool = Puppet::Network::HTTP::Pool.new(Puppet[:http_keepalive_timeout])
+      pool = Puppet.runtime['http'].pool
       Puppet.override(:http_pool => pool) do
         begin
           handler = Puppet::Configurer::PluginHandler.new()

data/lib/puppet/functions/eyaml_lookup_key.rb

@@ -39,7 +39,7 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
       context.cache(nil, raw_data)
     end
     context.not_found unless raw_data.include?(key)
-    context.cache(key, decrypt_value(raw_data[key], context, options))
+    context.cache(key, decrypt_value(raw_data[key], context, options, key))
   end
 
   def load_data_hash(options, context)
@@ -62,22 +62,22 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
     end
   end
 
-  def decrypt_value(value, context, options)
+  def decrypt_value(value, context, options, key)
     case value
     when String
-      decrypt(value, context, options)
+      decrypt(value, context, options, key)
     when Hash
       result = {}
-      value.each_pair { |k, v| result[context.interpolate(k)] = decrypt_value(v, context, options) }
+      value.each_pair { |k, v| result[context.interpolate(k)] = decrypt_value(v, context, options, key) }
       result
     when Array
-      value.map { |v| decrypt_value(v, context, options) }
+      value.map { |v| decrypt_value(v, context, options, key) }
     else
       value
     end
   end
 
-  def decrypt(data, context, options)
+  def decrypt(data, context, options, key)
     if encrypted?(data)
       # Options must be set prior to each call to #parse since they end up as static variables in
       # the Options class. They cannot be set once before #decrypt_value is called, since each #decrypt
@@ -85,8 +85,13 @@ Puppet::Functions.create_function(:eyaml_lookup_key) do
       # config.
       #
       Hiera::Backend::Eyaml::Options.set(options)
-      tokens = Hiera::Backend::Eyaml::Parser::ParserFactory.hiera_backend_parser.parse(data)
-      data = tokens.map(&:to_plain_text).join.chomp
+      begin
+        tokens = Hiera::Backend::Eyaml::Parser::ParserFactory.hiera_backend_parser.parse(data)
+        data = tokens.map(&:to_plain_text).join.chomp
+      rescue StandardError => ex
+        raise Puppet::DataBinding::LookupError,
+          _("hiera-eyaml backend error decrypting %{data} when looking up %{key} in %{path}. Error was %{message}") % { data: data, key: key, path: options['path'], message: ex.message }
+      end
     end
     context.interpolate(data)
   end

data/lib/puppet/http.rb

@@ -13,6 +13,7 @@ module Puppet
 
   module HTTP
     ACCEPT_ENCODING = "gzip;q=1.0,deflate;q=0.6,identity;q=0.3".freeze
+    HEADER_PUPPET_VERSION = "X-Puppet-Version".freeze
 
     require 'puppet/http/errors'
     require 'puppet/http/response'

data/lib/puppet/http/client.rb

@@ -1,11 +1,14 @@
 class Puppet::HTTP::Client
-  def initialize(pool: Puppet::Network::HTTP::Pool.new, ssl_context: nil, redirect_limit: 10, retry_limit: 100)
+  attr_reader :pool
+
+  def initialize(pool: Puppet::Network::HTTP::Pool.new, ssl_context: nil, system_ssl_context: nil, redirect_limit: 10, retry_limit: 100)
     @pool = pool
     @default_headers = {
       'X-Puppet-Version' => Puppet.version,
       'User-Agent' => Puppet[:http_user_agent],
     }.freeze
     @default_ssl_context = ssl_context
+    @default_system_ssl_context = system_ssl_context
     @redirector = Puppet::HTTP::Redirector.new(redirect_limit)
     @retry_after_handler = Puppet::HTTP::RetryAfterHandler.new(retry_limit, Puppet[:runinterval])
     @resolvers = build_resolvers
@@ -15,11 +18,15 @@ class Puppet::HTTP::Client
     Puppet::HTTP::Session.new(self, @resolvers)
   end
 
-  def connect(uri, ssl_context: nil, &block)
+  def connect(uri, ssl_context: nil, include_system_store: false, &block)
     start = Time.now
-    ctx = ssl_context ? ssl_context : default_ssl_context
+    ctx = resolve_ssl_context(ssl_context, include_system_store)
     site = Puppet::Network::HTTP::Site.from_uri(uri)
-    verifier = Puppet::SSL::Verifier.new(site.host, ctx)
+    verifier = if site.use_ssl?
+                 Puppet::SSL::Verifier.new(site.host, ctx)
+               else
+                 nil
+               end
     connected = false
 
     @pool.with_connection(site, verifier) do |http|
@@ -43,7 +50,7 @@ class Puppet::HTTP::Client
                 {uri: uri, elapsed: elapsed(start), message: e.message}, e, connected)
   end
 
-  def get(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil, &block)
+  def get(url, headers: {}, params: {}, user: nil, password: nil, ssl_context: nil, include_system_store: false, &block)
     query = encode_params(params)
     unless query.empty?
       url = url.dup
@@ -52,16 +59,16 @@ class Puppet::HTTP::Client
 
     request = Net::HTTP::Get.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+    execute_streaming(request, user: user, password: password, ssl_context: ssl_context, include_system_store: include_system_store) do |response|
       if block_given?
         yield response
       else
-        response.read_body
+        response.body
       end
     end
   end
 
-  def head(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
+  def head(url, headers: {}, params: {}, user: nil, password: nil, ssl_context: nil, include_system_store: false)
     query = encode_params(params)
     unless query.empty?
       url = url.dup
@@ -70,12 +77,12 @@ class Puppet::HTTP::Client
 
     request = Net::HTTP::Head.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    execute_streaming(request, user: user, password: password, ssl_context: ssl_context, include_system_store: include_system_store) do |response|
+      response.body
     end
   end
 
-  def put(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil)
+  def put(url, headers: {}, params: {}, content_type:, body:, user: nil, password: nil, ssl_context: nil, include_system_store: false)
     query = encode_params(params)
     unless query.empty?
       url = url.dup
@@ -87,12 +94,12 @@ class Puppet::HTTP::Client
     request['Content-Length'] = body.bytesize
     request['Content-Type'] = content_type
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    execute_streaming(request, user: user, password: password, ssl_context: ssl_context, include_system_store: include_system_store) do |response|
+      response.body
     end
   end
 
-  def post(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil, &block)
+  def post(url, headers: {}, params: {}, content_type:, body:, user: nil, password: nil, ssl_context: nil, include_system_store: false, &block)
     query = encode_params(params)
     unless query.empty?
       url = url.dup
@@ -104,16 +111,16 @@ class Puppet::HTTP::Client
     request['Content-Length'] = body.bytesize
     request['Content-Type'] = content_type
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+    execute_streaming(request, user: user, password: password, ssl_context: ssl_context, include_system_store: include_system_store) do |response|
       if block_given?
         yield response
       else
-        response.read_body
+        response.body
       end
     end
   end
 
-  def delete(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
+  def delete(url, headers: {}, params: {}, user: nil, password: nil, ssl_context: nil, include_system_store: false)
     query = encode_params(params)
     unless query.empty?
       url = url.dup
@@ -122,8 +129,8 @@ class Puppet::HTTP::Client
 
     request = Net::HTTP::Delete.new(url, @default_headers.merge(headers))
 
-    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
-      response.read_body
+    execute_streaming(request, user: user, password: password, ssl_context: ssl_context, include_system_store: include_system_store) do |response|
+      response.body
     end
   end
 
@@ -133,16 +140,19 @@ class Puppet::HTTP::Client
 
   private
 
-  def execute_streaming(request, ssl_context:, user: nil, password: nil, &block)
+  def execute_streaming(request, user: nil, password: nil, ssl_context:, include_system_store:, &block)
     redirects = 0
     retries = 0
+    response = nil
+    done = false
 
-    loop do
-      connect(request.uri, ssl_context: ssl_context) do |http|
+    while !done do
+      connect(request.uri, ssl_context: ssl_context, include_system_store: include_system_store) do |http|
         apply_auth(request, user, password)
 
+        # don't call return within the `request` block
         http.request(request) do |nethttp|
-          response = Puppet::HTTP::Response.new(nethttp)
+          response = Puppet::HTTP::Response.new(nethttp, request.uri)
           begin
             Puppet.debug("HTTP #{request.method.upcase} #{request.uri} returned #{response.code} #{response.reason}")
 
@@ -154,6 +164,10 @@ class Puppet::HTTP::Client
               interval = @retry_after_handler.retry_after_interval(request, response, retries)
               retries += 1
               if interval
+                if http.started?
+                  Puppet.debug("Closing connection for #{Puppet::Network::HTTP::Site.from_uri(request.uri)}")
+                  http.finish
+                end
                 Puppet.warning(_("Sleeping for %{interval} seconds before retrying the request") % { interval: interval })
                 ::Kernel.sleep(interval)
                 next
@@ -165,10 +179,12 @@ class Puppet::HTTP::Client
             response.drain
           end
 
-          return response
+          done = true
         end
       end
     end
+
+    response
   end
 
   def expand_into_parameters(data)
@@ -219,8 +235,25 @@ class Puppet::HTTP::Client
     end
   end
 
-  def default_ssl_context
-    @default_ssl_context || Puppet.lookup(:ssl_context)
+  def resolve_ssl_context(ssl_context, include_system_store)
+    if ssl_context
+      raise Puppet::HTTP::HTTPError, "The ssl_context and include_system_store parameters are mutually exclusive" if include_system_store
+      ssl_context
+    elsif include_system_store
+      system_ssl_context
+    else
+      @default_ssl_context || Puppet.lookup(:ssl_context)
+    end
+  end
+
+  def system_ssl_context
+    return @default_system_ssl_context if @default_system_ssl_context
+
+    cert_provider = Puppet::X509::CertProvider.new
+    cacerts = cert_provider.load_cacerts || []
+
+    ssl = Puppet::SSL::SSLProvider.new
+    @default_system_ssl_context = ssl.create_system_context(cacerts: cacerts)
   end
 
   def apply_auth(request, user, password)
@@ -238,14 +271,16 @@ class Puppet::HTTP::Client
 
     server_list_setting = Puppet.settings.setting(:server_list)
     if server_list_setting.value && !server_list_setting.value.empty?
-      services = [:puppet]
-
-      # If we have not explicitly set :ca_server either on the command line or
-      # in puppet.conf, we want to be able to try the servers defined by
-      # :server_list when resolving the :ca service. Otherwise, :server_list
-      # should only be used with the :puppet service.
-      if !Puppet.settings.set_by_config?(:ca_server)
-        services << :ca
+      # use server list to resolve all services
+      services = Puppet::HTTP::Service::SERVICE_NAMES.dup
+
+      # except if it's been explicitly set
+      if Puppet.settings.set_by_config?(:ca_server)
+        services.delete(:ca)
+      end
+
+      if Puppet.settings.set_by_config?(:report_server)
+        services.delete(:report)
       end
 
       resolvers << Puppet::HTTP::Resolver::ServerList.new(self, server_list_setting: server_list_setting, default_port: Puppet[:masterport], services: services)

data/lib/puppet/http/resolver/server_list.rb

@@ -13,7 +13,7 @@ class Puppet::HTTP::Resolver::ServerList < Puppet::HTTP::Resolver
         port = server[1] || @default_port
         uri = URI("https://#{host}:#{port}/status/v1/simple/master")
         if get_success?(uri, session, ssl_context: ssl_context)
-          return Puppet::HTTP::Service.create_service(@client, name, host, port)
+          return Puppet::HTTP::Service.create_service(@client, session, name, host, port)
         end
       end
       raise Puppet::Error, _("Could not select a functional puppet master from server_list: '%{server_list}'") % { server_list: @server_list_setting.print(@server_list_setting.value) }
@@ -27,7 +27,7 @@ class Puppet::HTTP::Resolver::ServerList < Puppet::HTTP::Resolver
     return true if response.success?
 
     Puppet.debug(_("Puppet server %{host}:%{port} is unavailable: %{code} %{reason}") %
-                 { host: host, port: port, code: response.code, reason: response.message })
+                 { host: uri.host, port: uri.port, code: response.code, reason: response.reason })
     return false
   rescue => detail
     session.add_exception(detail)

data/lib/puppet/http/resolver/settings.rb

@@ -1,6 +1,6 @@
 class Puppet::HTTP::Resolver::Settings < Puppet::HTTP::Resolver
   def resolve(session, name, ssl_context: nil)
-    service = Puppet::HTTP::Service.create_service(@client, name)
+    service = Puppet::HTTP::Service.create_service(@client, session, name)
     check_connection?(session, service, ssl_context: ssl_context) ? service : nil
   end
 end

data/lib/puppet/http/resolver/srv.rb

@@ -10,7 +10,7 @@ class Puppet::HTTP::Resolver::SRV < Puppet::HTTP::Resolver
     # This is fine for :ca, but note that :puppet and :file are handled
     # specially in `each_srv_record`.
     @delegate.each_srv_record(@srv_domain, name) do |server, port|
-      service = Puppet::HTTP::Service.create_service(@client, name, server, port)
+      service = Puppet::HTTP::Service.create_service(@client, session, name, server, port)
       return service if check_connection?(session, service, ssl_context: ssl_context)
     end
 

data/lib/puppet/http/response.rb

@@ -1,6 +1,9 @@
 class Puppet::HTTP::Response
-  def initialize(nethttp)
+  attr_reader :nethttp, :url
+
+  def initialize(nethttp, url)
     @nethttp = nethttp
+    @url = url
   end
 
   def code
@@ -16,6 +19,8 @@ class Puppet::HTTP::Response
   end
 
   def read_body(&block)
+    raise ArgumentError, "A block is required" unless block_given?
+
     @nethttp.read_body(&block)
   end
 

data/lib/puppet/http/service.rb

@@ -4,16 +4,16 @@ class Puppet::HTTP::Service
   SERVICE_NAMES = [:ca, :fileserver, :puppet, :report].freeze
   EXCLUDED_FORMATS = [:yaml, :b64_zlib_yaml, :dot].freeze
 
-  def self.create_service(client, name, server = nil, port = nil)
+  def self.create_service(client, session, name, server = nil, port = nil)
     case name
     when :ca
-      Puppet::HTTP::Service::Ca.new(client, server, port)
+      Puppet::HTTP::Service::Ca.new(client, session, server, port)
     when :fileserver
-      Puppet::HTTP::Service::FileServer.new(client, server, port)
+      Puppet::HTTP::Service::FileServer.new(client, session, server, port)
     when :puppet
-      ::Puppet::HTTP::Service::Compiler.new(client, server, port)
+      ::Puppet::HTTP::Service::Compiler.new(client, session, server, port)
     when :report
-      Puppet::HTTP::Service::Report.new(client, server, port)
+      Puppet::HTTP::Service::Report.new(client, session, server, port)
     else
       raise ArgumentError, "Unknown service #{name}"
     end
@@ -23,8 +23,9 @@ class Puppet::HTTP::Service
     SERVICE_NAMES.include?(name)
   end
 
-  def initialize(client, url)
+  def initialize(client, session, url)
     @client = client
+    @session = session
     @url = url
   end
 
@@ -42,7 +43,22 @@ class Puppet::HTTP::Service
 
   def add_puppet_headers(headers)
     modified_headers = headers.dup
+
+    # Add 'X-Puppet-Profiling' to enable performance profiling if turned on
     modified_headers['X-Puppet-Profiling'] = 'true' if Puppet[:profile]
+
+    # Add additional user-defined headers if they are defined
+    Puppet[:http_extra_headers].each do |name, value|
+      if modified_headers.keys.find { |key| key.casecmp(name) == 0 }
+        Puppet.warning(_('Ignoring extra header "%{name}" as it was previously set.') % { name: name })
+      else
+        if value.nil? || value.empty?
+          Puppet.warning(_('Ignoring extra header "%{name}" as it has no value.') % { name: name })
+        else
+          modified_headers[name] = value
+        end
+      end
+    end
     modified_headers
   end
 
@@ -54,11 +70,8 @@ class Puppet::HTTP::Service
   end
 
   def get_mime_types(model)
-    unless @mime_types
-      network_formats = model.supported_formats - EXCLUDED_FORMATS
-      @mime_types = network_formats.map { |f| model.get_format(f).mime }
-    end
-    @mime_types
+    network_formats = model.supported_formats - EXCLUDED_FORMATS
+    network_formats.map { |f| model.get_format(f).mime }
   end
 
   def formatter_for_response(response)
@@ -106,4 +119,10 @@ class Puppet::HTTP::Service
       raise Puppet::HTTP::SerializationError.new("Failed to deserialize multiple #{model} from #{formatter.name}: #{err.message}", err)
     end
   end
+
+  def process_response(response)
+    @session.process_response(response)
+
+    raise Puppet::HTTP::ResponseError.new(response) unless response.success?
+  end
 end