puppet 6.11.1 → 6.12.0

data/lib/puppet/environments.rb

@@ -1,3 +1,5 @@
+require 'puppet/concurrent/synchronized'
+
 # @api private
 module Puppet::Environments
 
@@ -293,6 +295,7 @@ module Puppet::Environments
 
   class Cached
     include EnvironmentLoader
+    include Puppet::Concurrent::Synchronized
 
     class DefaultCacheExpirationService
       def created(env)

data/lib/puppet/error.rb

@@ -12,7 +12,7 @@ module Puppet
     # This module implements logging with a filename and line number. Use this
     # for errors that need to report a location in a non-ruby file that we
     # parse.
-    attr_accessor :line, :file, :pos
+    attr_accessor :line, :file, :pos, :puppetstack
 
     # May be called with 3 arguments for message, file, line, and exception, or
     # 4 args including the position on the line.
@@ -22,10 +22,18 @@ module Puppet
         original = pos
         pos = nil
       end
+
       super(message, original)
+
       @file = file unless (file.is_a?(String) && file.empty?)
       @line = line
       @pos = pos
+
+      if original && original.respond_to?(:puppetstack)
+        @puppetstack = original.puppetstack
+      else
+        @puppetstack = Puppet::Pops::PuppetStack.stacktrace()
+      end
     end
 
     def to_s

data/lib/puppet/forge.rb

@@ -65,7 +65,7 @@ class Puppet::Forge < SemanticPuppet::Dependency::Source
       # make_http_request URI encodes parameters
       response = make_http_request(uri)
 
-      if response.code == '200'
+      if response.code == 200
         result = Puppet::Util::Json.load(response.body)
         uri = decode_uri(result['pagination']['next'])
         matches.concat result['results']
@@ -102,7 +102,7 @@ class Puppet::Forge < SemanticPuppet::Dependency::Source
       # make_http_request URI encodes parameters
       response = make_http_request(uri)
 
-      if response.code == '200'
+      if response.code == 200
         response = Puppet::Util::Json.load(response.body)
       else
         raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :response => response)
@@ -207,7 +207,7 @@ class Puppet::Forge < SemanticPuppet::Dependency::Source
     def download(uri, destination)
       response = @source.make_http_request(uri, destination)
       destination.flush and destination.close
-      unless response.code == '200'
+      unless response.code == 200
         raise Puppet::Forge::Errors::ResponseError.new(:uri => uri, :response => response)
       end
     end

data/lib/puppet/forge/errors.rb

@@ -74,12 +74,12 @@ module Puppet::Forge::Errors
     # @option options [String] :uri The URI that failed
     # @option options [String] :input The user's input (e.g. module name)
     # @option options [String] :message Error from the API response (optional)
-    # @option options [Net::HTTPResponse] :response The original HTTP response
+    # @option options [Puppet::HTTP::Response] :response The original HTTP response
     def initialize(options)
       @uri     = options[:uri]
       @message = options[:message]
       response = options[:response]
-      @response = "#{response.code} #{response.message.strip}"
+      @response = "#{response.code} #{response.reason.strip}"
 
       begin
         body = Puppet::Util::Json.load(response.body)

data/lib/puppet/forge/repository.rb

@@ -15,24 +15,6 @@ class Puppet::Forge
 
     attr_reader :uri, :cache
 
-    # List of Net::HTTP exceptions to catch
-    NET_HTTP_EXCEPTIONS = [
-      EOFError,
-      Errno::ECONNABORTED,
-      Errno::ECONNREFUSED,
-      Errno::ECONNRESET,
-      Errno::EINVAL,
-      Errno::ETIMEDOUT,
-      Net::HTTPBadResponse,
-      Net::HTTPHeaderSyntaxError,
-      Net::ProtocolError,
-      SocketError,
-    ]
-
-    if Puppet.features.zlib?
-      NET_HTTP_EXCEPTIONS << Zlib::GzipFile::Error
-    end
-
     # Instantiate a new repository instance rooted at the +url+.
     # The library will report +for_agent+ in the User-Agent to the repository.
     def initialize(host, for_agent)
@@ -40,13 +22,40 @@ class Puppet::Forge
       @agent = for_agent
       @cache = Cache.new(self)
       @uri   = URI.parse(host)
+
+      ssl_provider = Puppet::SSL::SSLProvider.new
+      @ssl_context = ssl_provider.create_system_context(cacerts: [])
     end
 
     # Return a Net::HTTPResponse read for this +path+.
     def make_http_request(path, io = nil)
-      request = get_request_object(@uri.path.chomp('/')+path)
-      Puppet.debug "HTTP GET #{@host}#{request.path}"
-      return read_response(request, io)
+      raise ArgumentError, "Path must start with forward slash" unless path.start_with?('/')
+      begin
+        str = @uri.to_s
+        str.chomp!('/')
+        str += Puppet::Util.uri_encode(path)
+        uri = URI(str)
+
+        headers = { "User-Agent" => user_agent }
+        user = nil
+        password = nil
+
+        if forge_authorization
+          headers["Authorization"] = forge_authorization
+        elsif @uri.user && @uri.password
+          user = @uri.user
+          password = @uri.password
+        end
+
+        http = Puppet.runtime['http']
+        response = http.get(uri, headers: headers, user: user, password: password, ssl_context: @ssl_context)
+        io.write(response.body) if io.respond_to?(:write)
+        response
+      rescue Puppet::SSL::CertVerifyError => e
+        raise SSLVerifyError.new(:uri => @uri.to_s, :original => e.cause)
+      rescue => e
+        raise CommunicationError.new(:uri => @uri.to_s, :original => e)
+      end
     end
 
     def forge_authorization
@@ -57,71 +66,6 @@ class Puppet::Forge
       end
     end
 
-    # responsible for properly encoding a URI
-    def get_request_object(path)
-      headers = {
-        "User-Agent" => user_agent,
-      }
-
-      if Puppet.features.zlib?
-        headers = headers.merge({
-          "Accept-Encoding" => Puppet::Network::HTTP::Compression::ACCEPT_ENCODING
-        })
-      end
-
-      if forge_authorization
-        headers = headers.merge({"Authorization" => forge_authorization})
-      end
-
-      request = Net::HTTP::Get.new(Puppet::Util.uri_encode(path), headers)
-
-      unless @uri.user.nil? || @uri.password.nil? || forge_authorization
-        request.basic_auth(@uri.user, @uri.password)
-      end
-
-      return request
-    end
-
-    # Return a Net::HTTPResponse read from this HTTPRequest +request+.
-    #
-    # @param request [Net::HTTPRequest] request to make
-    # @return [Net::HTTPResponse] response from request
-    # @raise [Puppet::Forge::Errors::CommunicationError] if there is a network
-    #   related error
-    # @raise [Puppet::Forge::Errors::SSLVerifyError] if there is a problem
-    #  verifying the remote SSL certificate
-    def read_response(request, io = nil)
-      http_object = Puppet::Util::HttpProxy.get_http_object(uri)
-
-      http_object.start do |http|
-        response = http.request(request)
-
-        if Puppet.features.zlib?
-          if response && response.key?("content-encoding")
-            case response["content-encoding"]
-            when "gzip"
-              response.body = Zlib::GzipReader.new(StringIO.new(response.read_body), :encoding => "ASCII-8BIT").read
-              response.delete("content-encoding")
-            when "deflate"
-              response.body = Zlib::Inflate.inflate(response.read_body)
-              response.delete("content-encoding")
-            end
-          end
-        end
-
-        io.write(response.body) if io.respond_to? :write
-        response
-      end
-    rescue *NET_HTTP_EXCEPTIONS => e
-      raise CommunicationError.new(:uri => @uri.to_s, :original => e)
-    rescue OpenSSL::SSL::SSLError => e
-      if e.message =~ /certificate verify failed/
-        raise SSLVerifyError.new(:uri => @uri.to_s, :original => e)
-      else
-        raise e
-      end
-    end
-
     # Return the local file name containing the data downloaded from the
     # repository at +release+ (e.g. "myuser-mymodule").
     def retrieve(release)

data/lib/puppet/functions/camelcase.rb

@@ -23,8 +23,8 @@
 #
 # @example Camelcase of strings in an Array
 # ```puppet
-# ['abc_def', 'bcd_xyz'].capitalize()
-# capitalize(['abc_def', 'bcd_xyz'])
+# ['abc_def', 'bcd_xyz'].camelcase()
+# camelcase(['abc_def', 'bcd_xyz'])
 # ```
 # Would both result in `['AbcDef', 'BcdXyz']`
 #

data/lib/puppet/functions/epp.rb

@@ -6,12 +6,12 @@
 # The first argument to this function should be a `<MODULE NAME>/<TEMPLATE FILE>`
 # reference, which loads `<TEMPLATE FILE>` from `<MODULE NAME>`'s `templates`
 # directory. In most cases, the last argument is optional; if used, it should be a
-# [hash](/puppet/latest/reference/lang_data_hash.html) that contains parameters to
+# [hash](https://puppet.com/docs/puppet/latest/lang_data_hash.html) that contains parameters to
 # pass to the template.
 #
-# - See the [template](/puppet/latest/reference/lang_template.html) documentation
-# for general template usage information.
-# - See the [EPP syntax](/puppet/latest/reference/lang_template_epp.html)
+# - See the [template](https://puppet.com/docs/puppet/latest/lang_template.html)
+# documentation for general template usage information.
+# - See the [EPP syntax](https://puppet.com/docs/puppet/latest/lang_template_epp.html)
 # documentation for examples of EPP.
 #
 # For example, to call the apache module's `templates/vhost/_docroot.epp`

data/lib/puppet/functions/find_file.rb

@@ -1,19 +1,19 @@
 # Finds an existing file from a module and returns its path.
 #
-# The argument to this function should be a String as a `<MODULE NAME>/<FILE>`
-# reference, which will search for `<FILE>` relative to a module's `files`
+# This function accepts an argument that is a String as a `<MODULE NAME>/<FILE>`
+# reference, which searches for `<FILE>` relative to a module's `files`
 # directory. (For example, the reference `mysql/mysqltuner.pl` will search for the
 # file `<MODULES DIRECTORY>/mysql/files/mysqltuner.pl`.)
 #
 # This function can also accept:
-# 
-# * An absolute String path, which will check for the existence of a file from anywhere on disk.
-# * Multiple String arguments, which will return the path of the **first** file
-#   found, skipping non existing files.
-# * An array of string paths, which will return the path of the **first** file
-#   found from the given paths in the array, skipping non existing files.
 #
-# The function returns `undef` if none of the given paths were found
+# * An absolute String path, which checks for the existence of a file from anywhere on disk.
+# * Multiple String arguments, which returns the path of the **first** file
+#   found, skipping nonexistent files.
+# * An array of string paths, which returns the path of the **first** file
+#   found from the given paths in the array, skipping nonexistent files.
+#
+# The function returns `undef` if none of the given paths were found.
 #
 # @since 4.8.0
 #

data/lib/puppet/functions/find_template.rb

@@ -0,0 +1,63 @@
+# Finds an existing template from a module and returns its path.
+#
+# This function accepts an argument that is a String as a `<MODULE NAME>/<TEMPLATE>`
+# reference, which searches for `<TEMPLATE>` relative to a module's `templates`
+# directory on the master. (For example, the reference `mymod/secret.conf.epp`
+# will search for the file `<MODULES DIRECTORY>/mymod/templates/secret.conf.epp`.)
+#
+# The primary use case is for agent-side template rendering with late-bound variables
+# resolved, such as from secret stores inaccessible to the master, such as
+#
+# ```
+# $variables = {
+#   'password' => Deferred('vault_lookup::lookup',
+#                   ['secret/mymod', 'https://vault.example.com:8200']),
+# }
+#
+# # compile the template source into the catalog
+# file { '/etc/secrets.conf':
+#   ensure  => file,
+#   content => Deferred('inline_epp',
+#                [find_template('mymod/secret.conf.epp').file, $variables]),
+# }
+# ```
+#
+#
+#
+# This function can also accept:
+#
+# * An absolute String path, which checks for the existence of a template from anywhere on disk.
+# * Multiple String arguments, which returns the path of the **first** template
+#   found, skipping nonexistent files.
+# * An array of string paths, which returns the path of the **first** template
+#   found from the given paths in the array, skipping nonexistent files.
+#
+# The function returns `undef` if none of the given paths were found.
+#
+# @since 6.x
+#
+Puppet::Functions.create_function(:find_template, Puppet::Functions::InternalFunction) do
+  dispatch :find_template do
+    scope_param
+    repeated_param 'String', :paths
+  end
+
+  dispatch :find_template_array do
+    scope_param
+    repeated_param 'Array[String]', :paths_array
+  end
+
+  def find_template_array(scope, array)
+    find_template(scope, *array)
+  end
+
+  def find_template(scope, *args)
+    args.each do |file|
+      found = Puppet::Parser::Files.find_template(file, scope.compiler.environment)
+      if found && Puppet::FileSystem.exist?(found)
+        return found
+      end
+    end
+    nil
+  end
+end

data/lib/puppet/functions/inline_epp.rb

@@ -5,12 +5,12 @@
 #
 # The first argument to this function should be a string containing an EPP
 # template. In most cases, the last argument is optional; if used, it should be a
-# [hash](/puppet/latest/reference/lang_data_hash.html) that contains parameters to
+# [hash](https://puppet.com/docs/puppet/latest/lang_data_hash.html) that contains parameters to
 # pass to the template.
 #
-# - See the [template](/puppet/latest/reference/lang_template.html) documentation
-# for general template usage information.
-# - See the [EPP syntax](/puppet/latest/reference/lang_template_epp.html)
+# - See the [template](https://puppet.com/docs/puppet/latest/lang_template.html)
+# documentation for general template usage information.
+# - See the [EPP syntax](https://puppet.com/docs/puppet/latest/lang_template_epp.html)
 # documentation for examples of EPP.
 #
 # For example, to evaluate an inline EPP template and pass it the `docroot` and
@@ -28,7 +28,7 @@
 # `inline_epp` function fails to pass any required parameter.
 #
 # An inline EPP template should be written as a single-quoted string or
-# [heredoc](/puppet/latest/reference/lang_data_string.html#heredocs).
+# [heredoc](https://puppet.com/docs/puppet/latest/lang_data_string.html#heredocs).
 # A double-quoted string is subject to expression interpolation before the string
 # is parsed as an EPP template.
 #

data/lib/puppet/http.rb

@@ -18,8 +18,10 @@ module Puppet
     require 'puppet/http/response'
     require 'puppet/http/service'
     require 'puppet/http/service/ca'
+    require 'puppet/http/service/report'
     require 'puppet/http/session'
     require 'puppet/http/resolver'
+    require 'puppet/http/resolver/server_list'
     require 'puppet/http/resolver/settings'
     require 'puppet/http/resolver/srv'
     require 'puppet/http/client'

data/lib/puppet/http/client.rb

@@ -16,19 +16,31 @@ class Puppet::HTTP::Client
   end
 
   def connect(uri, ssl_context: nil, &block)
+    start = Time.now
     ctx = ssl_context ? ssl_context : default_ssl_context
     site = Puppet::Network::HTTP::Site.from_uri(uri)
     verifier = Puppet::SSL::Verifier.new(site.host, ctx)
+    connected = false
 
     @pool.with_connection(site, verifier) do |http|
+      connected = true
       if block_given?
-        handle_post_connect(uri, http, &block)
+        yield http
       end
     end
+  rescue Net::OpenTimeout => e
+    raise_error(_("Request to %{uri} timed out connect operation after %{elapsed} seconds") % {uri: uri, elapsed: elapsed(start)}, e, connected)
+  rescue Net::ReadTimeout => e
+    raise_error(_("Request to %{uri} timed out read operation after %{elapsed} seconds") % {uri: uri, elapsed: elapsed(start)}, e, connected)
+  rescue EOFError => e
+    raise_error(_("Request to %{uri} interrupted after %{elapsed} seconds") % {uri: uri, elapsed: elapsed(start)}, e, connected)
+  rescue Puppet::SSL::SSLError
+    raise
   rescue Puppet::HTTP::HTTPError
     raise
   rescue => e
-    raise Puppet::HTTP::ConnectionError.new(_("Failed to connect to %{uri}: %{message}") % {uri: uri, message: e.message}, e)
+    raise_error(_("Request to %{uri} failed after %{elapsed} seconds: %{message}") %
+                {uri: uri, elapsed: elapsed(start), message: e.message}, e, connected)
   end
 
   def get(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil, &block)
@@ -49,6 +61,20 @@ class Puppet::HTTP::Client
     end
   end
 
+  def head(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
+    query = encode_params(params)
+    unless query.empty?
+      url = url.dup
+      url.query = query
+    end
+
+    request = Net::HTTP::Head.new(url, @default_headers.merge(headers))
+
+    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+      response.read_body
+    end
+  end
+
   def put(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil)
     query = encode_params(params)
     unless query.empty?
@@ -66,6 +92,41 @@ class Puppet::HTTP::Client
     end
   end
 
+  def post(url, headers: {}, params: {}, content_type:, body:, ssl_context: nil, user: nil, password: nil, &block)
+    query = encode_params(params)
+    unless query.empty?
+      url = url.dup
+      url.query = query
+    end
+
+    request = Net::HTTP::Post.new(url, @default_headers.merge(headers))
+    request.body = body
+    request['Content-Length'] = body.bytesize
+    request['Content-Type'] = content_type
+
+    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+      if block_given?
+        yield response
+      else
+        response.read_body
+      end
+    end
+  end
+
+  def delete(url, headers: {}, params: {}, ssl_context: nil, user: nil, password: nil)
+    query = encode_params(params)
+    unless query.empty?
+      url = url.dup
+      url.query = query
+    end
+
+    request = Net::HTTP::Delete.new(url, @default_headers.merge(headers))
+
+    execute_streaming(request, ssl_context: ssl_context, user: user, password: password) do |response|
+      response.read_body
+    end
+  end
+
   def close
     @pool.close
   end
@@ -116,23 +177,18 @@ class Puppet::HTTP::Client
     end.join('&')
   end
 
-  def handle_post_connect(uri, http, &block)
-    start = Time.now
-    yield http
-  rescue Puppet::HTTP::HTTPError
-    raise
-  rescue EOFError => e
-    raise Puppet::HTTP::HTTPError.new(_("Request to %{uri} interrupted after %{elapsed} seconds") % {uri: uri, elapsed: elapsed(start)}, e)
-  rescue Timeout::Error => e
-    raise Puppet::HTTP::HTTPError.new(_("Request to %{uri} timed out after %{elapsed} seconds") % {uri: uri, elapsed: elapsed(start)}, e)
-  rescue => e
-    raise Puppet::HTTP::HTTPError.new(_("Request to %{uri} failed after %{elapsed} seconds: %{message}") % {uri: uri, elapsed: elapsed(start), message: e.message}, e)
-  end
-
   def elapsed(start)
     (Time.now - start).to_f.round(3)
   end
 
+  def raise_error(message, cause, connected)
+    if connected
+      raise Puppet::HTTP::HTTPError.new(message, cause)
+    else
+      raise Puppet::HTTP::ConnectionError.new(message, cause)
+    end
+  end
+
   def default_ssl_context
     @default_ssl_context || Puppet.lookup(:ssl_context)
   end
@@ -147,10 +203,26 @@ class Puppet::HTTP::Client
     resolvers = []
 
     if Puppet[:use_srv_records]
-      resolvers << Puppet::HTTP::Resolver::SRV.new(domain: Puppet[:srv_domain])
+      resolvers << Puppet::HTTP::Resolver::SRV.new(self, domain: Puppet[:srv_domain])
     end
 
-    resolvers << Puppet::HTTP::Resolver::Settings.new
+    server_list_setting = Puppet.settings.setting(:server_list)
+    if server_list_setting.value && !server_list_setting.value.empty?
+      services = [:puppet]
+
+      # If we have not explicitly set :ca_server either on the command line or
+      # in puppet.conf, we want to be able to try the servers defined by
+      # :server_list when resolving the :ca service. Otherwise, :server_list
+      # should only be used with the :puppet service.
+      if !Puppet.settings.set_by_config?(:ca_server)
+        services << :ca
+      end
+
+      resolvers << Puppet::HTTP::Resolver::ServerList.new(self, server_list_setting: server_list_setting, default_port: Puppet[:masterport], services: services)
+    end
+
+    resolvers << Puppet::HTTP::Resolver::Settings.new(self)
+
     resolvers.freeze
   end
 end