puppet 4.0.0 → 4.5.0

data/lib/puppet/module_tool/tar/mini.rb

@@ -25,13 +25,13 @@ class Puppet::ModuleTool::Tar::Mini
   #
   # This check was mainly added to ignore 'x' and 'g' flags from the PAX
   # standard but will also ignore any other non-standard tar flags.
-  # tar format info: http://pic.dhe.ibm.com/infocenter/zos/v1r13/index.jsp?topic=%2Fcom.ibm.zos.r13.bpxa500%2Ftaf.htm
-  # pax format info: http://pic.dhe.ibm.com/infocenter/zos/v1r13/index.jsp?topic=%2Fcom.ibm.zos.r13.bpxa500%2Fpxarchfm.htm
+  # tar format info: https://pic.dhe.ibm.com/infocenter/zos/v1r13/index.jsp?topic=%2Fcom.ibm.zos.r13.bpxa500%2Ftaf.htm
+  # pax format info: https://pic.dhe.ibm.com/infocenter/zos/v1r13/index.jsp?topic=%2Fcom.ibm.zos.r13.bpxa500%2Fpxarchfm.htm
   def find_valid_files(tarfile)
     Archive::Tar::Minitar.open(tarfile).collect do |entry|
       flag = entry.typeflag
       if flag.nil? || flag =~ /[[:digit:]]/ && (0..7).include?(flag.to_i)
-        entry.name
+        entry.full_name
       else
         Puppet.debug "Invalid tar flag '#{flag}' will not be extracted: #{entry.name}"
         next

data/lib/puppet/network/auth_config_parser.rb

@@ -4,7 +4,7 @@ module Puppet::Network
 class AuthConfigParser
 
   def self.new_from_file(file)
-    self.new(File.read(file))
+    self.new(Puppet::FileSystem.read(file, :encoding => 'utf-8'))
   end
 
   def initialize(string)

data/lib/puppet/network/authconfig.rb

@@ -3,7 +3,7 @@ require 'puppet/network/http'
 
 module Puppet
   class ConfigurationError < Puppet::Error; end
-  class Network::AuthConfig
+  class Network::DefaultAuthProvider
     attr_accessor :rights
 
     def self.master_url_prefix
@@ -89,4 +89,24 @@ module Puppet
       insert_default_acl
     end
   end
+
+  class Network::AuthConfig
+    @@authprovider_class = nil
+
+    def self.authprovider_class=(klass)
+      @@authprovider_class = klass
+    end
+
+    def self.authprovider_class
+      @@authprovider_class || Puppet::Network::DefaultAuthProvider
+    end
+
+    def initialize(rights=nil)
+      @authprovider = self.class.authprovider_class.new(rights)
+    end
+
+    def check_authorization(method, path, params)
+      @authprovider.check_authorization(method, path, params)
+    end
+  end
 end

data/lib/puppet/network/authorization.rb

@@ -21,8 +21,15 @@ module Puppet::Network
   end
 
   module Authorization
+    @@authconfigloader_class = nil
+
+    def self.authconfigloader_class=(klass)
+      @@authconfigloader_class = klass
+    end
+
     def authconfig
-      AuthConfigLoader.authconfig
+      authconfigloader = @@authconfigloader_class || AuthConfigLoader
+      authconfigloader.authconfig
     end
 
     # Verify that our client has access.

data/lib/puppet/network/http/api/indirected_routes.rb

@@ -104,8 +104,6 @@ class Puppet::Network::HTTP::API::IndirectedRoutes
       raise ArgumentError, "No request key specified in #{uri}"
     end
 
-    key = URI.unescape(key)
-
     [indirection, method, key, params]
   end
 

data/lib/puppet/network/http/api/master/v3/environment.rb

@@ -0,0 +1,63 @@
+require 'json'
+require 'puppet/parser/environment_compiler'
+
+class Puppet::Network::HTTP::API::Master::V3::Environment
+  def call(request, response)
+    env_name = request.routing_path.split('/').last
+    env = Puppet.lookup(:environments).get(env_name)
+    code_id = request.params[:code_id]
+
+    if env.nil?
+      raise Puppet::Network::HTTP::Error::HTTPNotFoundError.new("#{env_name} is not a known environment", Puppet::Network::HTTP::Issues::RESOURCE_NOT_FOUND)
+    end
+
+    catalog = Puppet::Parser::EnvironmentCompiler.compile(env, code_id).to_resource
+
+    # This reads code_id off the catalog rather than using the one from the
+    # request. There shouldn't really be a case where the two differ, but if
+    # they do, the one from the catalog itself is authoritative.
+    env_graph = {:environment => env.name, :applications => {}, :code_id => catalog.code_id}
+    applications = catalog.resources.select do |res|
+      type = res.resource_type
+      type.is_a?(Puppet::Resource::Type) && type.application?
+    end
+    applications.each do |app|
+      app_components = {}
+      # Turn the 'nodes' hash into a map component ref => node name
+      node_mapping = {}
+      app['nodes'].each do |node, comps|
+        comps = [comps] unless comps.is_a?(Array)
+        comps.each do |comp|
+          raise Puppet::ParseError, "Application #{app} maps component #{comp} to multiple nodes" if node_mapping.include?(comp.ref)
+          node_mapping[comp.ref] = node.title
+        end
+      end
+
+      catalog.direct_dependents_of(app).each do |comp|
+        mapped_node = node_mapping[comp.ref]
+        if mapped_node.nil?
+          raise Puppet::ParseError, "Component #{comp} is not mapped to any node"
+        end
+        app_components[comp.ref] = {
+          :produces => comp.export.map(&:ref),
+          :consumes => prerequisites(comp).map(&:ref),
+          :node => mapped_node
+        }
+      end
+      env_graph[:applications][app.ref] = app_components
+    end
+    response.respond_with(200, "application/json", JSON.dump(env_graph))
+  end
+
+  private
+
+  # Finds all the prerequisites of component +comp+. They are all the
+  # capability resources that +comp+ depends on; this includes resources
+  # that +comp+ consumes but also resources it merely requires
+  def prerequisites(comp)
+    params = Puppet::Type.relationship_params.select { |p| p.direction == :in }.map(&:name)
+    params.map { |rel| comp[rel] }.flatten.compact.select do |rel|
+      rel.resource_type && rel.resource_type.is_capability?
+    end
+  end
+end

data/lib/puppet/network/http/api/master/v3.rb

@@ -1,6 +1,7 @@
 class Puppet::Network::HTTP::API::Master::V3
   require 'puppet/network/http/api/master/v3/authorization'
   require 'puppet/network/http/api/master/v3/environments'
+  require 'puppet/network/http/api/master/v3/environment'
   require 'puppet/network/http/api/indirected_routes'
 
   AUTHZ = Authorization.new
@@ -14,9 +15,14 @@ class Puppet::Network::HTTP::API::Master::V3
     Environments.new(Puppet.lookup(:environments))
   end)
 
+  ENVIRONMENT = Puppet::Network::HTTP::Route.
+      path(%r{/environment/[^/]+$}).get(AUTHZ.wrap do
+    Environment.new
+  end)
+
   def self.routes
     Puppet::Network::HTTP::Route.path(%r{v3}).
         any.
-        chain(ENVIRONMENTS, INDIRECTED)
+        chain(ENVIRONMENTS, ENVIRONMENT, INDIRECTED)
   end
 end

data/lib/puppet/network/http/factory.rb

@@ -1,5 +1,6 @@
 require 'openssl'
 require 'net/http'
+require 'puppet/util/http_proxy'
 
 # Factory for <tt>Net::HTTP</tt> objects.
 #
@@ -23,10 +24,13 @@ class Puppet::Network::HTTP::Factory
     Puppet.debug("Creating new connection for #{site}")
 
     args = [site.host, site.port]
-    if Puppet[:http_proxy_host] == "none"
-      args << nil << nil
-    else
-      args << Puppet[:http_proxy_host] << Puppet[:http_proxy_port]
+
+    unless Puppet::Util::HttpProxy.no_proxy?(site)
+      if Puppet[:http_proxy_host] == "none"
+        args << nil << nil
+      else
+        args << Puppet[:http_proxy_host] << Puppet[:http_proxy_port]
+      end
     end
 
     http = Net::HTTP.new(*args)

data/lib/puppet/network/http/pool.rb

@@ -81,10 +81,19 @@ class Puppet::Network::HTTP::Pool
 
       Puppet.debug("Starting connection for #{site}")
       http.start
+      setsockopts(http.instance_variable_get(:@socket))
       http
     end
   end
 
+  # Set useful socket option(s) which lack from default settings in Net:HTTP
+  #
+  # @api private
+  def setsockopts(netio)
+    socket = netio.io
+    socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+  end
+
   # Release a connection back into the pool.
   #
   # @api private

data/lib/puppet/network/http/rack/rest.rb

@@ -2,6 +2,7 @@ require 'openssl'
 require 'cgi'
 require 'puppet/network/http/handler'
 require 'puppet/util/ssl'
+require 'uri'
 
 class Puppet::Network::HTTP::RackREST
   include Puppet::Network::HTTP::Handler
@@ -79,7 +80,15 @@ class Puppet::Network::HTTP::RackREST
 
   # what path was requested? (this is, without any query parameters)
   def path(request)
-    request.path
+    # The value that Passenger provides for 'path' is escaped
+    # (URL percent-encoded), see
+    # https://github.com/phusion/passenger/blob/release-5.0.26/src/apache2_module/Hooks.cpp#L885
+    # for the implementation as hooked up to an Apache web server.  Code
+    # in the indirector / HTTP layer which consumes this path, however, assumes
+    # that it has already been unescaped, so it is unescaped here.
+    if request.path
+      URI.unescape(request.path)
+    end
   end
 
   # return the request body

data/lib/puppet/node/environment.rb

@@ -179,14 +179,30 @@ class Puppet::Node::Environment
   #   Puppet[:disable_per_environment_manifest] is true, and this environment's
   #   original environment.conf had a manifest setting that is not the
   #   Puppet[:default_manifest].
-  # @api private 
+  # @api private
   def conflicting_manifest_settings?
     return false if !Puppet[:disable_per_environment_manifest]
-    environment_conf = Puppet.lookup(:environments).get_conf(name)
-    original_manifest = environment_conf.raw_setting(:manifest)
+    original_manifest = configuration.raw_setting(:manifest)
     !original_manifest.nil? && !original_manifest.empty? && original_manifest != Puppet[:default_manifest]
   end
 
+  # @api private
+  def static_catalogs?
+    if @static_catalogs.nil?
+      environment_conf = Puppet.lookup(:environments).get_conf(name)
+      @static_catalogs = (environment_conf.nil? ? Puppet[:static_catalogs] : environment_conf.static_catalogs)
+    end
+    @static_catalogs
+  end
+
+  # Return the environment configuration
+  # @return [Puppet::Settings::EnvironmentConf] The configuration
+  #
+  # @api private
+  def configuration
+    Puppet.lookup(:environments).get_conf(name)
+  end
+
   # Checks the environment and settings for any conflicts
   # @return [Array<String>] an array of validation errors
   # @api public
@@ -318,13 +334,17 @@ class Puppet::Node::Environment
   def modules_by_path
     modules_by_path = {}
     modulepath.each do |path|
-      Dir.chdir(path) do
-        module_names = Dir.entries(path).select do |name|
-          Puppet::Module.is_module_directory?(name, path)
-        end
-        modules_by_path[path] = module_names.sort.map do |name|
-          Puppet::Module.new(name, File.join(path, name), self)
+      if Puppet::FileSystem.exist?(path)
+        Dir.chdir(path) do
+          module_names = Dir.entries(path).select do |name|
+            Puppet::Module.is_module_directory?(name, path)
+          end
+          modules_by_path[path] = module_names.sort.map do |name|
+            Puppet::Module.new(name, File.join(path, name), self)
+          end
         end
+      else
+        modules_by_path[path] = []
       end
     end
     modules_by_path
@@ -438,8 +458,8 @@ class Puppet::Node::Environment
   private
 
   def self.extralibs()
-    if ENV["PUPPETLIB"]
-      split_path(ENV["PUPPETLIB"])
+    if Puppet::Util.get_env('PUPPETLIB')
+      split_path(Puppet::Util.get_env('PUPPETLIB'))
     else
       []
     end
@@ -488,6 +508,10 @@ class Puppet::Node::Environment
         parser.parse
       end
     end
+  rescue Puppet::ParseErrorWithIssue => detail
+    @known_resource_types.parse_failed = true
+    detail.environment = self.name
+    raise
   rescue => detail
     @known_resource_types.parse_failed = true
 

data/lib/puppet/node.rb

@@ -9,6 +9,9 @@ class Puppet::Node
   # the node sources.
   extend Puppet::Indirector
 
+  # Asymmetric serialization/deserialization required in this class via to/from datahash
+  include Puppet::Util::PsychSupport
+
   # Use the node source as the indirection terminus.
   indirects :node, :terminus_setting => :node_terminus, :doc => "Where to find node information.
     A node is composed of its name, its facts, and its environment."
@@ -16,13 +19,20 @@ class Puppet::Node
   attr_accessor :name, :classes, :source, :ipaddress, :parameters, :trusted_data, :environment_name
   attr_reader :time, :facts
 
-  def self.from_data_hash(data)
-    raise ArgumentError, "No name provided in serialized data" unless name = data['name']
+  attr_reader :server_facts
 
+  ENVIRONMENT = 'environment'.freeze
+
+  def initialize_from_hash(data)
+    @name       = data['name']       || (raise ArgumentError, "No name provided in serialized data")
+    @classes    = data['classes']    || []
+    @parameters = data['parameters'] || {}
+    @environment_name = data['environment']
+  end
+
+  def self.from_data_hash(data)
     node = new(name)
-    node.classes = data['classes']
-    node.parameters = data['parameters']
-    node.environment_name = data['environment']
+    node.initialize_from_hash(data)
     node
   end
 
@@ -40,7 +50,7 @@ class Puppet::Node
     if @environment
       @environment
     else
-      if env = parameters["environment"]
+      if env = parameters[ENVIRONMENT]
         self.environment = env
       elsif environment_name
         self.environment = environment_name
@@ -62,6 +72,13 @@ class Puppet::Node
     else
       @environment = env
     end
+
+    # Keep environment_name attribute and parameter in sync if they have been set
+    unless @environment.nil?
+      @parameters[ENVIRONMENT] = @environment.name.to_s if @parameters.include?(ENVIRONMENT)
+      self.environment_name = @environment.name if instance_variable_defined?(:@environment_name)
+    end
+    @environment
   end
 
   def has_environment_instance?
@@ -86,6 +103,8 @@ class Puppet::Node
 
     @facts = options[:facts]
 
+    @server_facts = {}
+
     if env = options[:environment]
       self.environment = env
     end
@@ -108,10 +127,22 @@ class Puppet::Node
   # Merge any random parameters into our parameter list.
   def merge(params)
     params.each do |name, value|
-      @parameters[name] = value unless @parameters.include?(name)
+      if @parameters.include?(name)
+        Puppet::Util::Warnings.warnonce("The node parameter '#{name}' for node '#{@name}' was already set to '#{@parameters[name]}'. It could not be set to '#{value}'")
+      else
+        @parameters[name] = value
+      end
     end
 
-    @parameters["environment"] ||= self.environment.name.to_s
+    @parameters[ENVIRONMENT] ||= self.environment.name.to_s
+  end
+
+  def add_server_facts(facts)
+    # Append the current environment to the list of server facts
+    @server_facts = facts.merge({ "environment" => self.environment.name.to_s})
+
+    # Merge the server facts into the parameters for the node
+    merge(facts)
   end
 
   # Calculate the list of names we might use for looking

data/lib/puppet/parser/ast/pops_bridge.rb

@@ -23,6 +23,11 @@ class Puppet::Parser::AST::PopsBridge
       Puppet::Pops::Model::ModelTreeDumper.new.dump(@value)
     end
 
+    def source_text
+      source_adapter = Puppet::Pops::Utils.find_closest_positioned(@value)
+      source_adapter ? source_adapter.extract_text() : nil
+    end
+
     def evaluate(scope)
       object = @@evaluator.evaluate(scope, @value)
       @@evaluator.convert_to_3x(object, scope)
@@ -83,8 +88,26 @@ class Puppet::Parser::AST::PopsBridge
           instantiate_HostClassDefinition(d, modname)
         when Puppet::Pops::Model::ResourceTypeDefinition
           instantiate_ResourceTypeDefinition(d, modname)
+        when Puppet::Pops::Model::CapabilityMapping
+          instantiate_CapabilityMapping(d, modname)
         when Puppet::Pops::Model::NodeDefinition
           instantiate_NodeDefinition(d, modname)
+        when Puppet::Pops::Model::SiteDefinition
+            instantiate_SiteDefinition(d, modname)
+        when Puppet::Pops::Model::FunctionDefinition
+          # The 3x logic calling this will not know what to do with the result, it is compacted away at the end
+          instantiate_FunctionDefinition(d, modname)
+          next
+        when Puppet::Pops::Model::TypeAlias
+          # The 3x logic calling this will not know what to do with the result, it is compacted away at the end
+          instantiate_TypeAlias(d, modname)
+          next
+        when Puppet::Pops::Model::TypeMapping
+          # The 3x logic calling this will not know what to do with the result, it is compacted away at the end
+          instantiate_TypeMapping(d, modname)
+          next
+        when Puppet::Pops::Model::Application
+          instantiate_ApplicationDefinition(d, modname)
         else
           raise Puppet::ParseError, "Internal Error: Unknown type of definition - got '#{d.class}'"
         end
@@ -170,7 +193,30 @@ class Puppet::Parser::AST::PopsBridge
     end
 
     def instantiate_ResourceTypeDefinition(o, modname)
-      Puppet::Resource::Type.new(:definition, o.name, @context.merge(args_from_definition(o, modname)))
+      instance = Puppet::Resource::Type.new(:definition, o.name, @context.merge(args_from_definition(o, modname)))
+      Puppet::Pops::Loaders.register_runtime3_type(instance.name, Puppet::Pops::Adapters::SourcePosAdapter.adapt(o).to_uri)
+      instance
+    end
+
+    def instantiate_CapabilityMapping(o, modname)
+      # Use an intermediate 'capability_mapping' type to pass this info to the compiler where the
+      # actual mapping takes place
+      Puppet::Resource::Type.new(:capability_mapping, "#{o.component} #{o.kind} #{o.capability}", { :arguments => {
+        'component'     => o.component,
+        'kind'          => o.kind,
+        'blueprint'     => {
+          :capability => o.capability,
+          :mappings   => o.mappings.reduce({}) do |memo, mapping|
+            memo[mapping.attribute_name] =
+              Expression.new(:value => mapping.value_expr)
+            memo
+          end
+      }}})
+    end
+
+    def instantiate_ApplicationDefinition(o, modname)
+      args = args_from_definition(o, modname)
+      Puppet::Resource::Type.new(:application, o.name, @context.merge(args))
     end
 
     def instantiate_NodeDefinition(o, modname)
@@ -183,47 +229,62 @@ class Puppet::Parser::AST::PopsBridge
       unless is_nop?(o.parent)
         args[:parent] = @ast_transformer.hostname(o.parent)
       end
+      args = @ast_transformer.merge_location(args, o)
 
       host_matches = @ast_transformer.hostname(o.host_matches)
-      @ast_transformer.merge_location(args, o)
       host_matches.collect do |name|
         Puppet::Resource::Type.new(:node, name, @context.merge(args))
       end
     end
 
+    def instantiate_SiteDefinition(o, modname)
+      args = { :module_name => modname }
+
+      unless is_nop?(o.body)
+        args[:code] = Expression.new(:value => o.body)
+      end
+
+      args = @ast_transformer.merge_location(args, o)
+      Puppet::Resource::Type.new(:site, 'site', @context.merge(args))
+    end
+
     # Propagates a found Function to the appropriate loader.
     # This is for 4x evaluator/loader
     #
     def instantiate_FunctionDefinition(function_definition, modname)
-      loaders = (Puppet.lookup(:loaders) { nil })
-      unless loaders
-        raise Puppet::ParseError, "Internal Error: Puppet Context ':loaders' missing - cannot define any functions"
-      end
-      loader =
-      if modname.nil? || modname == ""
-        # TODO : Later when functions can be private, a decision is needed regarding what that means.
-        #        A private environment loader could be used for logic outside of modules, then only that logic
-        #        would see the function.
-        #
-        # Use the private loader, this function may see the environment's dependencies (currently, all modules)
-        loaders.private_environment_loader()
-      else
-        # TODO : Later check if function is private, and then add it to
-        #        private_loader_for_module
-        #
-        loaders.public_loader_for_module(modname)
-      end
-      unless loader
-        raise Puppet::ParseError, "Internal Error: did not find public loader for module: '#{modname}'"
-      end
+      loader = Puppet::Pops::Loaders.find_loader(modname)
 
-      # Instantiate Function, and store it in the environment loader
+      # Instantiate Function, and store it in the loader
       typed_name, f = Puppet::Pops::Loader::PuppetFunctionInstantiator.create_from_model(function_definition, loader)
       loader.set_entry(typed_name, f, Puppet::Pops::Adapters::SourcePosAdapter.adapt(function_definition).to_uri)
 
       nil # do not want the function to inadvertently leak into 3x
     end
 
+    # Propagates a found TypeAlias to the appropriate loader.
+    # This is for 4x evaluator/loader
+    #
+    def instantiate_TypeAlias(type_alias, modname)
+      loader = Puppet::Pops::Loaders.find_loader(modname)
+
+      # Bind the type alias to the loader using the alias
+      Puppet::Pops::Loader::TypeDefinitionInstantiator.create_from_model(type_alias, loader)
+
+      nil # do not want the type alias to inadvertently leak into 3x
+    end
+
+    # Adds the TypeMapping to the ImplementationRegistry
+    # This is for 4x evaluator/loader
+    #
+    def instantiate_TypeMapping(type_mapping, modname)
+      loader = Puppet::Pops::Loaders.find_loader(modname)
+      tf = Puppet::Pops::Types::TypeParser.new
+      lhs = tf.interpret(type_mapping.type_expr, loader)
+      rhs = tf.interpret_any(type_mapping.mapping_expr, loader)
+      Puppet::Pops::Loaders.implementation_registry.register_type_mapping(lhs, rhs, loader)
+      nil
+    end
+
     def code()
       Expression.new(:value => @value)
     end

data/lib/puppet/parser/compiler/catalog_validator/env_relationship_validator.rb

@@ -0,0 +1,64 @@
+class Puppet::Parser::Compiler
+  # Validator that asserts that all capability resources that are referenced by 'consume' or 'require' has
+  # been exported by some other resource in the environment
+ class CatalogValidator::EnvironmentRelationshipValidator < CatalogValidator
+
+    def validate
+      assumed_exports = {}
+      exported = {}
+      catalog.resources.each do |resource|
+        next unless resource.is_a?(Puppet::Parser::Resource)
+        resource.eachparam do |param|
+          pclass = Puppet::Type.metaparamclass(param.name)
+          validate_relationship(resource, param, assumed_exports, exported) if !pclass.nil? && pclass < Puppet::Type::RelationshipMetaparam
+        end
+      end
+      assumed_exports.each_pair do |key, (param, cap)|
+        raise CatalogValidationError.new("Capability '#{cap}' referenced by '#{param.name}' is never exported", param.file, param.line) unless exported.include?(key)
+      end
+      nil
+    end
+
+    private
+
+    def validate_relationship(resource, param, assumed_exports, exported)
+      case param.name
+      when :require, :consume
+        add_capability_ref(param, param.value, assumed_exports)
+      when :export
+        add_exported(resource, param, param.value, exported)
+      end
+    end
+
+    def add_capability_ref(param, value, assumed_exports)
+      case value
+      when Array
+        value.each { |v| add_capability_ref(param, v, assumed_exports) }
+      when Puppet::Resource
+        rt = value.resource_type
+        unless rt.nil? || !rt.is_capability?
+          title_key = catalog.title_key_for_ref(value.ref)
+          assumed_exports[title_key] = [param, value]
+        end
+        nil
+      end
+    end
+
+    def add_exported(resource, param, value, hash)
+      case value
+      when Array
+        value.each { |v| add_exported(resource, param, v, hash) }
+      when Puppet::Resource
+        rt = value.resource_type
+        unless rt.nil? || !rt.is_capability?
+          title_key = catalog.title_key_for_ref(value.ref)
+          if hash.include?(title_key)
+            raise CatalogValidationError.new("'#{value}' is exported by both '#{hash[title_key]}' and '#{resource}'", param.file, param.line)
+          else
+            hash[title_key] = resource
+          end
+        end
+      end
+    end
+  end
+end