puppet 4.1.0 → 4.10.0

data/lib/puppet/network/http/pool.rb

@@ -81,10 +81,19 @@ class Puppet::Network::HTTP::Pool
 
       Puppet.debug("Starting connection for #{site}")
       http.start
+      setsockopts(http.instance_variable_get(:@socket))
       http
     end
   end
 
+  # Set useful socket option(s) which lack from default settings in Net:HTTP
+  #
+  # @api private
+  def setsockopts(netio)
+    socket = netio.io
+    socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+  end
+
   # Release a connection back into the pool.
   #
   # @api private

data/lib/puppet/network/http/rack/rest.rb

@@ -2,6 +2,7 @@ require 'openssl'
 require 'cgi'
 require 'puppet/network/http/handler'
 require 'puppet/util/ssl'
+require 'uri'
 
 class Puppet::Network::HTTP::RackREST
   include Puppet::Network::HTTP::Handler
@@ -79,7 +80,15 @@ class Puppet::Network::HTTP::RackREST
 
   # what path was requested? (this is, without any query parameters)
   def path(request)
-    request.path
+    # The value that Passenger provides for 'path' is escaped
+    # (URL percent-encoded), see
+    # https://github.com/phusion/passenger/blob/release-5.0.26/src/apache2_module/Hooks.cpp#L885
+    # for the implementation as hooked up to an Apache web server.  Code
+    # in the indirector / HTTP layer which consumes this path, however, assumes
+    # that it has already been unescaped, so it is unescaped here.
+    if request.path
+      URI.unescape(request.path)
+    end
   end
 
   # return the request body
@@ -94,7 +103,21 @@ class Puppet::Network::HTTP::RackREST
     # NOTE: The SSL_CLIENT_CERT environment variable will be the empty string
     # when Puppet agent nodes have not yet obtained a signed certificate.
     if cert.nil? || cert.empty?
-      nil
+      # When running with unicorn, the SSL_CLIENT_CERT variable is not available
+      # in the environment, therefore we have to pass a header: 'X-SSL-Client-Cert'
+      cert = request.env['HTTP_X_SSL_CLIENT_CERT']
+      if cert.nil? || cert.empty?
+        nil
+      else
+        # in contrast to the environment variable, the client cert is passed in
+        # as single string, therefore restore the certificate to a valid pem
+        # encoded certificate
+        cert.gsub!(/ /, "\n")
+        cert.gsub!(/BEGIN\nCERT/, "BEGIN CERT")
+        cert.gsub!(/END\nCERT/, "END CERT")
+        cert = Puppet::SSL::Certificate.from_instance(OpenSSL::X509::Certificate.new(cert))
+        cert
+      end
     else
       Puppet::SSL::Certificate.from_instance(OpenSSL::X509::Certificate.new(cert))
     end

data/lib/puppet/network/http/webrick.rb

@@ -73,7 +73,8 @@ class Puppet::Network::HTTP::WEBrick
     file = Puppet[:masterhttplog]
 
     # open the log manually to prevent file descriptor leak
-    file_io = ::File.open(file, "a+")
+    # webrick logged strings may contain UTF-8
+    file_io = ::File.open(file, "a+:UTF-8")
     file_io.sync = true
     if defined?(Fcntl::FD_CLOEXEC)
       file_io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)

data/lib/puppet/network/rights.rb

@@ -53,17 +53,7 @@ class Rights
 
     # if we end up here, then that means we either didn't match or failed, in any
     # case will return an error to the outside world
-    host_description = args[:node] ? "#{args[:node]}(#{args[:ip]})" : args[:ip]
-
-    msg = "#{host_description} access to #{name} [#{args[:method]}]"
-
-    if args[:authenticated]
-      msg += " authenticated "
-    end
-
-    if right
-      msg += " at #{right.file}:#{right.line}"
-    end
+    msg = "#{name} [#{args[:method]}]"
 
     AuthorizationError.new("Forbidden request: #{msg}")
   end

data/lib/puppet/node/environment.rb

@@ -48,7 +48,7 @@ class Puppet::Node::Environment
     obj.send(:initialize,
              name.intern,
              expand_dirs(extralibs() + modulepath),
-             manifest == NO_MANIFEST ? manifest : File.expand_path(manifest),
+             manifest == NO_MANIFEST ? manifest : Puppet::FileSystem.expand_path(manifest),
              config_version)
     obj
   end
@@ -166,6 +166,10 @@ class Puppet::Node::Environment
   #     (optional)
   attr_reader :config_version
 
+  # Cached loaders - management of value handled by Puppet::Pops::Loaders
+  # @api private
+  attr_accessor :loaders
+
   # Checks to make sure that this environment did not have a manifest set in
   # its original environment.conf if Puppet is configured with
   # +disable_per_environment_manifest+ set true.  If it did, the environment's
@@ -179,14 +183,30 @@ class Puppet::Node::Environment
   #   Puppet[:disable_per_environment_manifest] is true, and this environment's
   #   original environment.conf had a manifest setting that is not the
   #   Puppet[:default_manifest].
-  # @api private 
+  # @api private
   def conflicting_manifest_settings?
     return false if !Puppet[:disable_per_environment_manifest]
-    environment_conf = Puppet.lookup(:environments).get_conf(name)
-    original_manifest = environment_conf.raw_setting(:manifest)
+    original_manifest = configuration.raw_setting(:manifest)
     !original_manifest.nil? && !original_manifest.empty? && original_manifest != Puppet[:default_manifest]
   end
 
+  # @api private
+  def static_catalogs?
+    if @static_catalogs.nil?
+      environment_conf = Puppet.lookup(:environments).get_conf(name)
+      @static_catalogs = (environment_conf.nil? ? Puppet[:static_catalogs] : environment_conf.static_catalogs)
+    end
+    @static_catalogs
+  end
+
+  # Return the environment configuration
+  # @return [Puppet::Settings::EnvironmentConf] The configuration
+  #
+  # @api private
+  def configuration
+    Puppet.lookup(:environments).get_conf(name)
+  end
+
   # Checks the environment and settings for any conflicts
   # @return [Array<String>] an array of validation errors
   # @api public
@@ -198,6 +218,17 @@ class Puppet::Node::Environment
     errors
   end
 
+  # Checks if this environment permits use of rich data types in the catalog
+  # @return [Boolean] `true` if rich data is permitted.
+  # @api private
+  def rich_data?
+    if @rich_data.nil?
+      environment_conf = Puppet.lookup(:environments).get_conf(name)
+      @rich_data = (environment_conf.nil? ? Puppet[:rich_data] : environment_conf.rich_data)
+    end
+    @rich_data
+  end
+
   # Return an environment-specific Puppet setting.
   #
   # @api public
@@ -318,13 +349,17 @@ class Puppet::Node::Environment
   def modules_by_path
     modules_by_path = {}
     modulepath.each do |path|
-      Dir.chdir(path) do
-        module_names = Dir.entries(path).select do |name|
-          Puppet::Module.is_module_directory?(name, path)
-        end
-        modules_by_path[path] = module_names.sort.map do |name|
-          Puppet::Module.new(name, File.join(path, name), self)
+      if Puppet::FileSystem.exist?(path)
+        Dir.chdir(path) do
+          module_names = Dir.entries(path).select do |name|
+            Puppet::Module.is_module_directory?(name, path)
+          end
+          modules_by_path[path] = module_names.sort.map do |name|
+            Puppet::Module.new(name, File.join(path, name), self)
+          end
         end
+      else
+        modules_by_path[path] = []
       end
     end
     modules_by_path
@@ -438,8 +473,8 @@ class Puppet::Node::Environment
   private
 
   def self.extralibs()
-    if ENV["PUPPETLIB"]
-      split_path(ENV["PUPPETLIB"])
+    if Puppet::Util.get_env('PUPPETLIB')
+      split_path(Puppet::Util.get_env('PUPPETLIB'))
     else
       []
     end
@@ -447,7 +482,7 @@ class Puppet::Node::Environment
 
   def self.expand_dirs(dirs)
     dirs.collect do |dir|
-      File.expand_path(dir)
+      Puppet::FileSystem.expand_path(dir)
     end
   end
 
@@ -501,7 +536,7 @@ class Puppet::Node::Environment
     raise error
   end
 
-  # Return an empty toplevel hostclass to indicate that no file was loaded
+  # Return an empty top-level hostclass to indicate that no file was loaded
   #
   # @return [Puppet::Parser::AST::Hostclass]
   def empty_parse_result

data/lib/puppet/node/facts.rb

@@ -59,7 +59,7 @@ class Puppet::Node::Facts
     end
   end
 
-  # Sanitize fact values by converting everything not a string, boolean
+  # Sanitize fact values by converting everything not a string, Boolean
   # numeric, array or hash into strings.
   def sanitize
     values.each do |fact, value|

data/lib/puppet/node.rb

@@ -9,6 +9,9 @@ class Puppet::Node
   # the node sources.
   extend Puppet::Indirector
 
+  # Asymmetric serialization/deserialization required in this class via to/from datahash
+  include Puppet::Util::PsychSupport
+
   # Use the node source as the indirection terminus.
   indirects :node, :terminus_setting => :node_terminus, :doc => "Where to find node information.
     A node is composed of its name, its facts, and its environment."
@@ -18,13 +21,18 @@ class Puppet::Node
 
   attr_reader :server_facts
 
-  def self.from_data_hash(data)
-    raise ArgumentError, "No name provided in serialized data" unless name = data['name']
+  ENVIRONMENT = 'environment'.freeze
+
+  def initialize_from_hash(data)
+    @name       = data['name']       || (raise ArgumentError, "No name provided in serialized data")
+    @classes    = data['classes']    || []
+    @parameters = data['parameters'] || {}
+    @environment_name = data['environment']
+  end
 
+  def self.from_data_hash(data)
     node = new(name)
-    node.classes = data['classes']
-    node.parameters = data['parameters']
-    node.environment_name = data['environment']
+    node.initialize_from_hash(data)
     node
   end
 
@@ -42,7 +50,7 @@ class Puppet::Node
     if @environment
       @environment
     else
-      if env = parameters["environment"]
+      if env = parameters[ENVIRONMENT]
         self.environment = env
       elsif environment_name
         self.environment = environment_name
@@ -64,6 +72,13 @@ class Puppet::Node
     else
       @environment = env
     end
+
+    # Keep environment_name attribute and parameter in sync if they have been set
+    unless @environment.nil?
+      @parameters[ENVIRONMENT] = @environment.name.to_s if @parameters.include?(ENVIRONMENT)
+      self.environment_name = @environment.name if instance_variable_defined?(:@environment_name)
+    end
+    @environment
   end
 
   def has_environment_instance?
@@ -119,7 +134,7 @@ class Puppet::Node
       end
     end
 
-    @parameters["environment"] ||= self.environment.name.to_s
+    @parameters[ENVIRONMENT] ||= self.environment.name.to_s
   end
 
   def add_server_facts(facts)

data/lib/puppet/parameter/boolean.rb

@@ -1,6 +1,6 @@
 require 'puppet/coercion'
 
-# This specialized {Puppet::Parameter} handles boolean options, accepting lots
+# This specialized {Puppet::Parameter} handles Boolean options, accepting lots
 # of strings and symbols for both truthiness and falsehood.
 #
 class Puppet::Parameter::Boolean < Puppet::Parameter

data/lib/puppet/parameter/value.rb

@@ -11,7 +11,7 @@ class Puppet::Parameter::Value
 
   # Adds an alias for this value.
   # Makes the given _name_ be an alias for this acceptable value.
-  # @param name [Symbol] the additonal alias this value should be known as
+  # @param name [Symbol] the additional alias this value should be known as
   # @api private
   #
   def alias(name)

data/lib/puppet/parameter.rb

@@ -295,6 +295,10 @@ class Puppet::Parameter
   #
   attr_accessor :parent
 
+  # @!attribute [rw] sensitive
+  #   @return [true, false] If this parameter has been tagged as sensitive.
+  attr_accessor :sensitive
+
   # Returns a string representation of the resource's containment path in
   # the catalog.
   # @return [String]
@@ -376,7 +380,7 @@ class Puppet::Parameter
     tmp
   end
 
-  # Returns an array of strings representing the containment heirarchy
+  # Returns an array of strings representing the containment hierarchy
   # (types/classes) that make up the path to the resource from the root
   # of the catalog.  This is mostly used for logging purposes.
   #
@@ -479,7 +483,7 @@ class Puppet::Parameter
   #   calls both validate and munge on the given value, so no late binding.
   #
   # The given value is validated and then munged (if munging has been specified). The result is store
-  # as the value of this arameter.
+  # as the value of this parameter.
   # @return [Object] The given `value` after munging.
   # @raise (see #validate)
   #
@@ -501,7 +505,7 @@ class Puppet::Parameter
 
   # @return [Array<Symbol>] Returns an array of the associated resource's symbolic tags (including the parameter itself).
   # Returns an array of the associated resource's symbolic tags (including the parameter itself).
-  # At a minimun, the array contains the name of the parameter. If the associated resource
+  # At a minimum, the array contains the name of the parameter. If the associated resource
   # has tags, these tags are also included in the array.
   # @todo The original comment says = _"The properties need to return tags so that logs correctly
   #   collect them."_ what if anything of that is of interest to document. Should tags and their relationship
@@ -522,6 +526,22 @@ class Puppet::Parameter
     name.to_s
   end
 
+  # Formats the given string and conditionally redacts the provided interpolation variables, depending on if
+  # this property is sensitive.
+  #
+  # @note Because the default implementation of Puppet::Property#is_to_s returns the current value as-is, it
+  #   doesn't necessarily return a string. For the sake of sanity we just cast everything to a string for
+  #   interpolation so we don't introduce issues with unexpected property values.
+  #
+  # @see String#format
+  # @param fmt [String] The format string to interpolate.
+  # @param args [Array<String>] One or more strings to conditionally redact and interpolate into the format string.
+  #
+  # @return [String]
+  def format(fmt, *args)
+    fmt % args.map { |arg| @sensitive ? "[redacted]" : arg.to_s }
+  end
+
   # Produces a String with the value formatted for display to a human.
   # When the parameter value is a:
   #

data/lib/puppet/parser/ast/pops_bridge.rb

@@ -23,6 +23,11 @@ class Puppet::Parser::AST::PopsBridge
       Puppet::Pops::Model::ModelTreeDumper.new.dump(@value)
     end
 
+    def source_text
+      source_adapter = Puppet::Pops::Utils.find_closest_positioned(@value)
+      source_adapter ? source_adapter.extract_text() : nil
+    end
+
     def evaluate(scope)
       object = @@evaluator.evaluate(scope, @value)
       @@evaluator.convert_to_3x(object, scope)
@@ -56,6 +61,19 @@ class Puppet::Parser::AST::PopsBridge
     end
   end
 
+  class ExpressionSupportingReturn < Expression
+    def initialize args
+      super
+    end
+
+    def evaluate(scope)
+      return catch(:return) do
+        return catch(:next) do
+          return super(scope)
+        end
+      end
+    end
+  end
   # Bridges the top level "Program" produced by the pops parser.
   # Its main purpose is to give one point where all definitions are instantiated (actually defined since the
   # Puppet 3x terminology is somewhat misleading - the definitions are instantiated, but instances of the created types
@@ -83,12 +101,26 @@ class Puppet::Parser::AST::PopsBridge
           instantiate_HostClassDefinition(d, modname)
         when Puppet::Pops::Model::ResourceTypeDefinition
           instantiate_ResourceTypeDefinition(d, modname)
+        when Puppet::Pops::Model::CapabilityMapping
+          instantiate_CapabilityMapping(d, modname)
         when Puppet::Pops::Model::NodeDefinition
           instantiate_NodeDefinition(d, modname)
+        when Puppet::Pops::Model::SiteDefinition
+            instantiate_SiteDefinition(d, modname)
         when Puppet::Pops::Model::FunctionDefinition
+          # The 3x logic calling this will not know what to do with the result, it is compacted away at the end
           instantiate_FunctionDefinition(d, modname)
+          next
+        when Puppet::Pops::Model::TypeAlias
+          # The 3x logic calling this will not know what to do with the result, it is compacted away at the end
+          instantiate_TypeAlias(d, modname)
+          next
+        when Puppet::Pops::Model::TypeMapping
           # The 3x logic calling this will not know what to do with the result, it is compacted away at the end
+          instantiate_TypeMapping(d, modname)
           next
+        when Puppet::Pops::Model::Application
+          instantiate_ApplicationDefinition(d, modname)
         else
           raise Puppet::ParseError, "Internal Error: Unknown type of definition - got '#{d.class}'"
         end
@@ -155,26 +187,49 @@ class Puppet::Parser::AST::PopsBridge
     end
 
     # Produces a hash with data for Definition and HostClass
-    def args_from_definition(o, modname)
+    def args_from_definition(o, modname, expr_class = Expression)
       args = {
        :arguments => o.parameters.collect {|p| instantiate_Parameter(p) },
        :argument_types => create_type_map(o),
        :module_name => modname
       }
       unless is_nop?(o.body)
-        args[:code] = Expression.new(:value => o.body)
+        args[:code] = expr_class.new(:value => o.body)
       end
       @ast_transformer.merge_location(args, o)
     end
 
     def instantiate_HostClassDefinition(o, modname)
-      args = args_from_definition(o, modname)
+      args = args_from_definition(o, modname, ExpressionSupportingReturn)
       args[:parent] = absolute_reference(o.parent_class)
       Puppet::Resource::Type.new(:hostclass, o.name, @context.merge(args))
     end
 
     def instantiate_ResourceTypeDefinition(o, modname)
-      Puppet::Resource::Type.new(:definition, o.name, @context.merge(args_from_definition(o, modname)))
+      instance = Puppet::Resource::Type.new(:definition, o.name, @context.merge(args_from_definition(o, modname, ExpressionSupportingReturn)))
+      Puppet::Pops::Loaders.register_runtime3_type(instance.name, Puppet::Pops::Adapters::SourcePosAdapter.adapt(o).to_uri)
+      instance
+    end
+
+    def instantiate_CapabilityMapping(o, modname)
+      # Use an intermediate 'capability_mapping' type to pass this info to the compiler where the
+      # actual mapping takes place
+      Puppet::Resource::Type.new(:capability_mapping, "#{o.component} #{o.kind} #{o.capability}", { :arguments => {
+        'component'     => o.component,
+        'kind'          => o.kind,
+        'blueprint'     => {
+          :capability => o.capability,
+          :mappings   => o.mappings.reduce({}) do |memo, mapping|
+            memo[mapping.attribute_name] =
+              Expression.new(:value => mapping.value_expr)
+            memo
+          end
+      }}})
+    end
+
+    def instantiate_ApplicationDefinition(o, modname)
+      args = args_from_definition(o, modname)
+      Puppet::Resource::Type.new(:application, o.name, @context.merge(args))
     end
 
     def instantiate_NodeDefinition(o, modname)
@@ -187,47 +242,62 @@ class Puppet::Parser::AST::PopsBridge
       unless is_nop?(o.parent)
         args[:parent] = @ast_transformer.hostname(o.parent)
       end
+      args = @ast_transformer.merge_location(args, o)
 
       host_matches = @ast_transformer.hostname(o.host_matches)
-      @ast_transformer.merge_location(args, o)
       host_matches.collect do |name|
         Puppet::Resource::Type.new(:node, name, @context.merge(args))
       end
     end
 
+    def instantiate_SiteDefinition(o, modname)
+      args = { :module_name => modname }
+
+      unless is_nop?(o.body)
+        args[:code] = Expression.new(:value => o.body)
+      end
+
+      args = @ast_transformer.merge_location(args, o)
+      Puppet::Resource::Type.new(:site, 'site', @context.merge(args))
+    end
+
     # Propagates a found Function to the appropriate loader.
     # This is for 4x evaluator/loader
     #
     def instantiate_FunctionDefinition(function_definition, modname)
-      loaders = (Puppet.lookup(:loaders) { nil })
-      unless loaders
-        raise Puppet::ParseError, "Internal Error: Puppet Context ':loaders' missing - cannot define any functions"
-      end
-      loader =
-      if modname.nil? || modname == ""
-        # TODO : Later when functions can be private, a decision is needed regarding what that means.
-        #        A private environment loader could be used for logic outside of modules, then only that logic
-        #        would see the function.
-        #
-        # Use the private loader, this function may see the environment's dependencies (currently, all modules)
-        loaders.private_environment_loader()
-      else
-        # TODO : Later check if function is private, and then add it to
-        #        private_loader_for_module
-        #
-        loaders.public_loader_for_module(modname)
-      end
-      unless loader
-        raise Puppet::ParseError, "Internal Error: did not find public loader for module: '#{modname}'"
-      end
+      loader = Puppet::Pops::Loaders.find_loader(modname)
 
-      # Instantiate Function, and store it in the environment loader
+      # Instantiate Function, and store it in the loader
       typed_name, f = Puppet::Pops::Loader::PuppetFunctionInstantiator.create_from_model(function_definition, loader)
       loader.set_entry(typed_name, f, Puppet::Pops::Adapters::SourcePosAdapter.adapt(function_definition).to_uri)
 
       nil # do not want the function to inadvertently leak into 3x
     end
 
+    # Propagates a found TypeAlias to the appropriate loader.
+    # This is for 4x evaluator/loader
+    #
+    def instantiate_TypeAlias(type_alias, modname)
+      loader = Puppet::Pops::Loaders.find_loader(modname)
+
+      # Bind the type alias to the loader using the alias
+      Puppet::Pops::Loader::TypeDefinitionInstantiator.create_from_model(type_alias, loader)
+
+      nil # do not want the type alias to inadvertently leak into 3x
+    end
+
+    # Adds the TypeMapping to the ImplementationRegistry
+    # This is for 4x evaluator/loader
+    #
+    def instantiate_TypeMapping(type_mapping, modname)
+      loader = Puppet::Pops::Loaders.find_loader(modname)
+      tf = Puppet::Pops::Types::TypeParser.singleton
+      lhs = tf.interpret(type_mapping.type_expr, loader)
+      rhs = tf.interpret_any(type_mapping.mapping_expr, loader)
+      Puppet::Pops::Loaders.implementation_registry.register_type_mapping(lhs, rhs, loader)
+      nil
+    end
+
     def code()
       Expression.new(:value => @value)
     end

data/lib/puppet/parser/ast/resource.rb

@@ -5,6 +5,11 @@ class Puppet::Parser::AST::Resource < Puppet::Parser::AST::Branch
 
   attr_accessor :type, :instances, :exported, :virtual
 
+  def initialize(argshash)
+    Puppet.warn_once('deprecation', 'AST::Resource', 'Use of Puppet::Parser::AST::Resource is deprecated and not fully functional')
+    super(argshash)
+  end
+
   # Evaluates resources by adding them to the compiler for lazy evaluation
   # and returning the produced resource references.
   #

data/lib/puppet/parser/ast/resource_instance.rb

@@ -2,5 +2,9 @@
 # title and a set of parameters.
 #
 class Puppet::Parser::AST::ResourceInstance < Puppet::Parser::AST::Branch
-    attr_accessor :title, :parameters
+  attr_accessor :title, :parameters
+  def initialize(argshash)
+    Puppet.warn_once('deprecation', 'AST::ResourceInstance', 'Use of Puppet::Parser::AST::ResourceInstance is deprecated')
+    super(argshash)
+  end
 end

data/lib/puppet/parser/ast/resourceparam.rb

@@ -3,6 +3,11 @@
 class Puppet::Parser::AST::ResourceParam < Puppet::Parser::AST::Branch
   attr_accessor :value, :param, :add
 
+  def initialize(argshash)
+    Puppet.warn_once('deprecation', 'AST::ResourceParam', 'Use of Puppet::Parser::AST::ResourceParam is deprecated and not fully functional')
+    super(argshash)
+  end
+
   def each
     [@param, @value].each { |child| yield child }
   end

data/lib/puppet/parser/ast.rb

@@ -29,6 +29,11 @@ class Puppet::Parser::AST
     # is called so many times during parsing.
     begin
       return self.evaluate(scope)
+    rescue Puppet::Pops::Evaluator::PuppetStopIteration => detail
+      raise detail
+#      # Only deals with StopIteration from the break() function as a general
+#      # StopIteration is a general runtime problem
+#      raise Puppet::ParseError.new(detail.message, detail.file, detail.line, detail)
     rescue Puppet::Error => detail
       raise adderrorcontext(detail)
     rescue => detail