puppet 4.0.0 → 4.10.0

data/lib/puppet/indirector/request.rb

@@ -178,7 +178,7 @@ class Puppet::Indirector::Request
     return(uri ? uri : "/#{indirection_name}/#{key}")
   end
 
-  def do_request(srv_service=:puppet, default_server=Puppet.settings[:server], default_port=Puppet.settings[:masterport], &block)
+  def do_request(srv_service=:puppet, default_server=nil, default_port=nil, &block)
     # We were given a specific server to use, so just use that one.
     # This happens if someone does something like specifying a file
     # source using a puppet:// URI with a specific server.
@@ -197,9 +197,30 @@ class Puppet::Indirector::Request
     end
 
     # ... Fall back onto the default server.
-    Puppet.debug "No more servers left, falling back to #{default_server}:#{default_port}" if Puppet.settings[:use_srv_records]
-    self.server = default_server
-    self.port   = default_port
+    begin
+      bound_server = Puppet.lookup(:server)
+    rescue
+      if primary_server = Puppet.settings[:server_list][0]
+        bound_server = primary_server[0]
+      else
+        bound_server = nil
+      end
+    end
+
+    begin
+      bound_port = Puppet.lookup(:serverport)
+    rescue
+      if primary_server = Puppet.settings[:server_list][0]
+        bound_port = primary_server[1]
+      else
+        bound_port = nil
+      end
+    end
+    self.server = default_server || bound_server || Puppet.settings[:server]
+    self.port   = default_port || bound_port || Puppet.settings[:masterport]
+
+    Puppet.debug "No more servers left, falling back to #{self.server}:#{self.port}" if Puppet.settings[:use_srv_records]
+
     return yield(self)
   end
 
@@ -222,6 +243,8 @@ class Puppet::Indirector::Request
   def set_uri_key(key)
     @uri = key
     begin
+      # calling URI.escape for UTF-8 characters will % escape them
+      # and the resulting string components of the URI are now ASCII
       uri = URI.parse(URI.escape(key))
     rescue => detail
       raise ArgumentError, "Could not understand URL #{key}: #{detail}", detail.backtrace
@@ -243,15 +266,16 @@ class Puppet::Indirector::Request
       @port = uri.port.to_i
     end
 
-    @protocol = uri.scheme
-
-    if uri.scheme == 'puppet'
-      @key = URI.unescape(uri.path.sub(/^\//, ''))
-      return
+    # filebucket:// is only used internally to pass request details
+    # from Dipper objects to the indirector. The wire always uses HTTPS.
+    if uri.scheme == 'filebucket'
+      @protocol = 'https'
+    else
+      @protocol = uri.scheme
     end
 
-    env, indirector, @key = URI.unescape(uri.path.sub(/^\//, '')).split('/',3)
-    @key ||= ''
-    self.environment = env unless env == ''
+    # The unescaped bytes are correct but in ASCII and must be treated
+    # as UTF-8 for the sake of performing string comparisons later
+    @key = URI.unescape(uri.path.sub(/^\//, '')).force_encoding(Encoding::UTF_8)
   end
 end

data/lib/puppet/indirector/resource_type/parser.rb

@@ -27,7 +27,7 @@ class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code
       krt = resource_types_in(request.environment)
 
       # This is a bit ugly.
-      [:hostclass, :definition, :node].each do |type|
+      [:hostclass, :definition, :application, :node].each do |type|
         # We have to us 'find_<type>' here because it will
         # load any missing types from disk, whereas the plain
         # '<type>' method only returns from memory.
@@ -66,14 +66,16 @@ class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code
             krt.hostclasses.values
           when "defined_type"
             krt.definitions.values
+          when "application"
+            krt.applications.values
           when "node"
             krt.nodes.values
           when nil
-            result_candidates = [krt.hostclasses.values, krt.definitions.values, krt.nodes.values]
+            result_candidates = [krt.hostclasses.values, krt.definitions.values, krt.applications.values, krt.nodes.values]
           else
             raise ArgumentError, "Unrecognized kind filter: " +
                       "'#{request.options[:kind]}', expected one " +
-                      " of 'class', 'defined_type', or 'node'."
+                      " of 'class', 'defined_type', 'application', or 'node'."
         end
 
       result = result_candidates.flatten.reject { |t| t.name == "" }
@@ -98,4 +100,9 @@ class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code
     environment.check_for_reparse
     environment.known_resource_types
   end
+
+  def allow_remote_requests?
+    Puppet.deprecation_warning("The resource_type endpoint is deprecated in favor of the environment_classes endpoint. See https://docs.puppet.com/puppetserver/latest/puppet-api/v3/environment_classes.html")
+    super
+  end
 end

data/lib/puppet/indirector/rest.rb

@@ -1,5 +1,6 @@
 require 'net/http'
 require 'uri'
+require 'json'
 
 require 'puppet/network/http'
 require 'puppet/network/http_pool'
@@ -33,18 +34,78 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus
     @srv_service || :puppet
   end
 
+  # The logic for server and port is kind of gross. In summary:
+  # IF an endpoint-specific setting is requested AND that setting has been set by the user
+  #    Use that setting.
+  #         The defaults for these settings are the "normal" server/masterport settings, so
+  #         when they are unset we instead want to "fall back" to the failover-selected
+  #         host/port pair.
+  # ELSE IF we have a failover-selected host/port
+  #    Use what the failover logic came up with
+  # ELSE IF the server_list setting is in use
+  #    Use the first entry - failover hasn't happened yet, but that
+  #    setting is still authoritative
+  # ELSE
+  #    Go for the legacy server/masterport settings, and hope for the best
   def self.server
-    Puppet.settings[server_setting || :server]
+    setting = server_setting()
+    if setting && setting != :server && Puppet.settings.set_by_config?(setting)
+      Puppet.settings[setting]
+    else
+      begin
+        Puppet.lookup(:server)
+      rescue
+        if primary_server = Puppet.settings[:server_list][0]
+          Puppet.debug "Dynamically-bound server lookup failed; using first entry"
+          primary_server[0]
+        else
+          setting ||= :server
+          Puppet.debug "Dynamically-bound server lookup failed, falling back to #{setting} setting"
+          Puppet.settings[setting]
+        end
+      end
+    end
   end
 
+  # For port there's a little bit of an extra snag: setting a specific
+  # server setting and relying on the default port for that server is
+  # common, so we also want to check if the assocaited SERVER setting
+  # has been set by the user. If either of those are set we ignore the
+  # failover-selected port.
   def self.port
-    Puppet.settings[port_setting || :masterport].to_i
+    setting = port_setting()
+    srv_setting = server_setting()
+    if (setting && setting != :masterport && Puppet.settings.set_by_config?(setting)) ||
+       (srv_setting && srv_setting != :server && Puppet.settings.set_by_config?(srv_setting))
+      Puppet.settings[setting].to_i
+    else
+      begin
+        Puppet.lookup(:serverport).to_i
+      rescue
+        if primary_server = Puppet.settings[:server_list][0]
+          Puppet.debug "Dynamically-bound port lookup failed; using first entry"
+
+          # Port might not be set, so we want to fallback in that
+          # case. We know we don't need to use `setting` here, since
+          # the default value of every port setting is `masterport`
+          (primary_server[1] || Puppet.settings[:masterport]).to_i
+        else
+          setting ||= :masterport
+          Puppet.debug "Dynamically-bound port lookup failed; falling back to #{setting} setting"
+          Puppet.settings[setting].to_i
+        end
+      end
+    end
   end
 
   # Provide appropriate headers.
   def headers
+    # yaml is not allowed on the network
+    network_formats = model.supported_formats.reject do |format|
+      [:yaml, :b64_zlib_yaml].include?(format)
+    end
     common_headers = {
-      "Accept"                                     => model.supported_formats.join(", "),
+      "Accept"                                     => network_formats.join(", "),
       Puppet::Network::HTTP::HEADER_PUPPET_VERSION => Puppet.version
     }
 
@@ -96,7 +157,8 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus
       # WEBrick in Ruby 1.9.1 only supports up to 1024 character lines in an HTTP request
       # http://redmine.ruby-lang.org/issues/show/3991
       if "GET #{uri_with_query_string} HTTP/1.1\r\n".length > 1024
-        http_post(req, uri, body, headers)
+        uri_with_env = "#{uri}?environment=#{request.environment.name}"
+        http_post(req, uri_with_env, body, headers)
       else
         http_get(req, uri_with_query_string, headers)
       end
@@ -213,7 +275,20 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus
   end
 
   def convert_to_http_error(response)
-    message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
+    if response.body.to_s.empty? && response.respond_to?(:message)
+      returned_message = response.message
+    elsif response['content-type'].is_a?(String)
+      content_type, body = parse_response(response)
+      if content_type =~ /[pj]son/
+        returned_message = JSON.parse(body)["message"]
+      else
+        returned_message = uncompress_body(response)
+      end
+    else
+      returned_message = uncompress_body(response)
+    end
+
+    message = "Error #{response.code} on SERVER: #{returned_message}"
     Net::HTTPError.new(message, response)
   end
 

data/lib/puppet/indirector/ssl_file.rb

@@ -112,6 +112,8 @@ class Puppet::Indirector::SslFile < Puppet::Indirector::Terminus
 
   def create_model(name, path)
     result = model.new(name)
+    # calls Puppet::SSL::Base#read for subclasses of Puppet::SSL::Base
+    # with the exception of any overrides, like Puppet::SSL::Key
     result.read(path)
     result
   end
@@ -158,13 +160,32 @@ class Puppet::Indirector::SslFile < Puppet::Indirector::Terminus
   # Yield a filehandle set up appropriately, either with our settings doing
   # the work or opening a filehandle manually.
   def write(name, path)
+    # All types serialized to disk contain only ASCII content:
+    # * SSL::Key may be a .export(OpenSSL::Cipher::DES.new(:EDE3, :CBC), pass) or .to_pem
+    # * All other classes are translated to strings by calling .to_pem
+
+    # Serialization of:
+    # Puppet::SSL::Certificate by Puppet::SSL::Certificate::Ca, Puppet::SSL::Certificate::File
+    # -----BEGIN CERTIFICATE-----
+    # Puppet::SSL::Key by Puppet::SSL::Key::Ca, Puppet::SSL::Key::File
+    # -----BEGIN RSA PRIVATE KEY-----
     if ca?(name) and ca_location
-      Puppet.settings.setting(self.class.ca_setting).open('w') { |f| yield f }
+      Puppet.settings.setting(self.class.ca_setting).open('w:ASCII') { |f| yield f }
+    # Serialization of:
+    # Puppet::SSL::CertificateRevocationList by Puppet::SSL::CertificateRevocationList::Ca, Puppet::SSL::CertificateRevocationList::File
+    # -----BEGIN X509 CRL-----
     elsif file_location
-      Puppet.settings.setting(self.class.file_setting).open('w') { |f| yield f }
+      Puppet.settings.setting(self.class.file_setting).open('w:ASCII') { |f| yield f }
+    # Serialization of:
+    # Puppet::SSL::Certificate by Puppet::SSL::Certificate::Ca, Puppet::SSL::Certificate::File
+    # -----BEGIN CERTIFICATE-----
+    # Puppet::SSL::CertificateRequest by Puppet::SSL::CertificateRequest::Ca, Puppet::SSL::CertificateRequest::File
+    # -----BEGIN CERTIFICATE REQUEST-----
+    # Puppet::SSL::Key by Puppet::SSL::Key::Ca, Puppet::SSL::Key::File
+    # -----BEGIN RSA PRIVATE KEY-----
     elsif setting = self.class.directory_setting
       begin
-        Puppet.settings.setting(setting).open_file(path, 'w') { |f| yield f }
+        Puppet.settings.setting(setting).open_file(path, 'w:ASCII') { |f| yield f }
       rescue => detail
         raise Puppet::Error, "Could not write #{path} to #{setting}: #{detail}", detail.backtrace
       end

data/lib/puppet/indirector/terminus.rb

@@ -66,18 +66,18 @@ class Puppet::Indirector::Terminus
       # Yay, class/instance confusion.
       subclass.terminus_type = self.name
 
-      # Our subclass is specifically associated with an indirection.
+      # This subclass is specifically associated with an indirection.
       raise("Invalid name #{longname}") unless names.length > 0
-      indirection_name = names.pop.sub(/^[A-Z]/) { |i| i.downcase }.gsub(/[A-Z]/) { |i| "_#{i.downcase}" }.intern
+      processed_name = names.pop.sub(/^[A-Z]/) { |i| i.downcase }.gsub(/[A-Z]/) { |i| "_#{i.downcase}" }
 
-      if indirection_name == "" or indirection_name.nil?
+      if processed_name.empty?
         raise Puppet::DevError, "Could not discern indirection model from class constant"
       end
 
       # This will throw an exception if the indirection instance cannot be found.
       # Do this last, because it also registers the terminus type with the indirection,
       # which needs the above information.
-      subclass.indirection = indirection_name
+      subclass.indirection = processed_name.intern
 
       # And add this instance to the instance hash.
       Puppet::Indirector::Terminus.register_terminus_class(subclass)

data/lib/puppet/info_service/class_information_service.rb

@@ -0,0 +1,111 @@
+require 'puppet'
+require 'puppet/pops'
+require 'puppet/pops/evaluator/json_strict_literal_evaluator'
+
+class Puppet::InfoService::ClassInformationService
+
+  def initialize
+    @file_to_result = {}
+    @parser = Puppet::Pops::Parser::EvaluatingParser.new()
+  end
+
+  def classes_per_environment(env_file_hash)
+    # In this version of puppet there is only one way to parse manifests, as feature switches per environment
+    # are added or removed, this logic needs to change to compute the result per environment with the correct
+    # feature flags in effect.
+
+    unless env_file_hash.is_a?(Hash)
+      raise ArgumentError, 'Given argument must be a Hash'
+    end
+
+    result = {}
+
+    # for each environment
+    #   for each file
+    #     if file already processed, use last result or error
+    #
+    env_file_hash.each do |env, files|
+      env_result = result[env] = {}
+      files.each do |f|
+        env_result[f] = result_of(f)
+      end
+    end
+    result
+  end
+
+  private
+
+  def type_parser
+    Puppet::Pops::Types::TypeParser.singleton
+  end
+
+  def literal_evaluator
+    @@literal_evaluator ||= Puppet::Pops::Evaluator::JsonStrictLiteralEvaluator.new
+  end
+
+  def result_of(f)
+    entry =  @file_to_result[f]
+    if entry.nil?
+      @file_to_result[f] = entry = parse_file(f)
+    end
+    entry
+  end
+
+  def parse_file(f)
+    return {:error => "The file #{f} does not exist"} unless Puppet::FileSystem.exist?(f)
+
+    begin
+      parse_result = @parser.parse_file(f)
+      {:classes =>
+        parse_result.definitions.select {|d| d.is_a?(Puppet::Pops::Model::HostClassDefinition)}.map do |d|
+          {:name   => d.name,
+           :params => params = d.parameters.map {|p| extract_param(p) }
+          }
+        end
+      }
+    rescue StandardError => e
+      {:error => e.message }
+    end
+  end
+
+  def extract_param(p)
+    extract_default(extract_type({:name => p.name}, p), p)
+  end
+
+  def extract_type(structure, p)
+    return structure if p.type_expr.nil?
+    structure[:type] = typeexpr_to_string(p.type_expr)
+    structure
+  end
+
+  def extract_default(structure, p)
+    value_expr = p.value
+    return structure if value_expr.nil?
+    default_value = value_as_literal(value_expr)
+    structure[:default_literal] = default_value unless default_value.nil?
+    structure[:default_source] = extract_value_source(value_expr)
+    structure
+  end
+
+  def typeexpr_to_string(type_expr)
+    begin
+      type_parser.interpret_any(type_expr, nil).to_s
+    rescue Puppet::ParseError
+      # type is to complex - contains expressions that are not literal
+      nil
+    end
+  end
+
+  def value_as_literal(value_expr)
+    catch(:not_literal) do
+      return literal_evaluator.literal(value_expr)
+    end
+    nil
+  end
+
+  # Extracts the source for the expression
+  def extract_value_source(value_expr)
+    position = Puppet::Pops::Adapters::SourcePosAdapter.adapt(value_expr)
+    position.extract_tree_text
+  end
+end

data/lib/puppet/info_service.rb

@@ -0,0 +1,7 @@
+
+module Puppet::InfoService
+  require 'puppet/info_service/class_information_service'
+  def self.classes_per_environment(env_file_hash)
+    Puppet::InfoService::ClassInformationService.new.classes_per_environment(env_file_hash)
+  end
+end

data/lib/puppet/interface/documentation.rb

@@ -29,7 +29,7 @@ class Puppet::Interface
       # We use module_eval, which I don't like much, because we can't have an
       # argument to a block with a default value in Ruby 1.8, and I don't like
       # the side-effects (eg: no argument count validation) of using blocks
-      # without as metheds.  When we are 1.9 only (hah!) you can totally
+      # without as methods.  When we are 1.9 only (hah!) you can totally
       # replace this with some up-and-up define_method. --daniel 2011-04-29
       module_eval(<<-EOT, __FILE__, __LINE__ + 1)
         def #{name}(value = nil)

data/lib/puppet/interface/face_collection.rb

@@ -27,7 +27,7 @@ module Puppet::Interface::FaceCollection
       # ...we need to search for it bound to an o{lder,ther} version.  Since
       # we load all actions when the face is first references, this will be in
       # memory in the known set of versions of the face.
-      (@faces[name].keys - [ :current ]).sort.reverse.each do |vers|
+      (@faces[name].keys - [ :current ]).sort.reverse_each do |vers|
         break if action = @faces[name][vers].get_action(action_name)
       end
     end

data/lib/puppet/interface.rb

@@ -149,12 +149,15 @@ class Puppet::Interface
 
   # @api private
   def initialize(name, version, &block)
-    unless SemVer.valid?(version)
+    unless SemanticPuppet::Version.valid?(version)
       raise ArgumentError, "Cannot create face #{name.inspect} with invalid version number '#{version}'!"
     end
 
     @name    = Puppet::Interface::FaceCollection.underscorize(name)
-    @version = SemVer.new(version)
+
+    # SemVer is deprecated but in 4.x we must use it here (the attr_reader is public api). The
+    # extra boolean argument suppresses the deprecation warning.
+    @version = SemVer.new(version, true)
 
     # The few bits of documentation we actually demand.  The default license
     # is a favour to our end users; if you happen to get that in a core face
@@ -180,6 +183,15 @@ class Puppet::Interface
     "Puppet::Face[#{name.inspect}, #{version.inspect}]"
   end
 
+  # @return [void]
+  def deprecate
+    @deprecated = true
+  end
+
+  # @return [Boolean]
+  def deprecated?
+    @deprecated
+  end
   ########################################################################
   # Action decoration, whee!  You are not expected to care about this code,
   # which exists to support face building and construction.  I marked these

data/lib/puppet/loaders.rb

@@ -3,6 +3,7 @@ module Puppet
     require 'puppet/pops/loaders'
 
     module Loader
+      require 'puppet/pops/loader/typed_name'
       require 'puppet/pops/loader/loader'
       require 'puppet/pops/loader/base_loader'
       require 'puppet/pops/loader/gem_support'
@@ -10,9 +11,14 @@ module Puppet
       require 'puppet/pops/loader/dependency_loader'
       require 'puppet/pops/loader/null_loader'
       require 'puppet/pops/loader/static_loader'
+      require 'puppet/pops/loader/runtime3_type_loader'
       require 'puppet/pops/loader/ruby_function_instantiator'
+      require 'puppet/pops/loader/puppet_function_instantiator'
+      require 'puppet/pops/loader/type_definition_instantiator'
+      require 'puppet/pops/loader/puppet_resource_type_impl_instantiator'
       require 'puppet/pops/loader/loader_paths'
       require 'puppet/pops/loader/simple_environment_loader'
+      require 'puppet/pops/loader/predefined_loader'
     end
   end
 

data/lib/puppet/metatype/manager.rb

@@ -20,6 +20,16 @@ module Manager
     }
   end
 
+  # Clears any types that were used but absent when types were last loaded.
+  # @note Used after each catalog compile when always_retry_plugins is false
+  # @api private
+  #
+  def clear_misses
+    unless @types.nil?
+      @types.delete_if {|_, v| v.nil? }
+    end
+  end
+
   # Iterates over all already loaded Type subclasses.
   # @yield [t] a block receiving each type
   # @yieldparam t [Puppet::Type] each defined type
@@ -116,6 +126,10 @@ module Manager
     klass.providerloader.loadall Puppet.lookup(:current_environment)
     klass.providify unless klass.providers.empty?
 
+    loc = block_given? ? block.source_location : nil
+    uri = loc.nil? ? nil : URI("#{Puppet::Util.path_to_uri(loc[0])}?line=#{loc[1]}")
+    Puppet::Pops::Loaders.register_runtime3_type(name, uri)
+
     klass
   end
 
@@ -146,16 +160,19 @@ module Manager
     # We are overwhelmingly symbols here, which usually match, so it is worth
     # having this special-case to return quickly.  Like, 25K symbols vs. 300
     # strings in this method. --daniel 2012-07-17
-    return @types[name] if @types[name]
+    return @types[name] if @types.include? name
 
     # Try mangling the name, if it is a string.
     if name.is_a? String
       name = name.downcase.intern
-      return @types[name] if @types[name]
+      return @types[name] if @types.include? name
     end
     # Try loading the type.
     if typeloader.load(name, Puppet.lookup(:current_environment))
       Puppet.warning "Loaded puppet/type/#{name} but no class was created" unless @types.include? name
+    elsif !Puppet[:always_retry_plugins]
+      # PUP-5482 - Only look for a type once if plugin retry is disabled
+      @types[name] = nil
     end
 
     # ...and I guess that is that, eh.
@@ -166,13 +183,13 @@ module Manager
   # Defaults to an instance of {Puppet::Util::Autoload} if no other auto loader has been set.
   # @return [Puppet::Util::Autoload] the loader to use.
   # @api private
-  def typeloader
-    unless defined?(@typeloader)
-      @typeloader = Puppet::Util::Autoload.new(self, "puppet/type")
-    end
-
-    @typeloader
+def typeloader
+  unless defined?(@typeloader)
+    @typeloader = Puppet::Util::Autoload.new(self, "puppet/type")
   end
+
+  @typeloader
+end
 end
 end