puppet 4.1.0 → 4.10.0

data/lib/puppet/ssl/oids.rb

@@ -13,6 +13,7 @@ require 'puppet/ssl'
 # -- privateExtensions can be extended by enterprises to suit their own needs
 # registeredExtensions OBJECT IDENTIFIER ::= { puppetCertExtensions 1 }
 # privateExtensions OBJECT IDENTIFIER ::= { puppetCertExtensions 2 }
+# authorizationExtensions OBJECT IDENTIFIER ::= { puppetCertExtensions 3 }
 #
 # -- subtree of common registered extensions
 # -- The short names for these OIDs are intentionally lowercased and formatted
@@ -25,6 +26,10 @@ require 'puppet/ssl'
 # @api private
 module Puppet::SSL::Oids
 
+  # Note: When updating the following OIDs make sure to also update the OID
+  # definitions here:
+  # https://github.com/puppetlabs/puppetserver/blob/master/src/clj/puppetlabs/puppetserver/certificate_authority.clj#L122-L159
+
   PUPPET_OIDS = [
     ["1.3.6.1.4.1.34380", 'puppetlabs', 'Puppet Labs'],
     ["1.3.6.1.4.1.34380.1", 'ppCertExt', 'Puppet Certificate Extension'],
@@ -48,8 +53,21 @@ module Puppet::SSL::Oids
     ["1.3.6.1.4.1.34380.1.1.15", 'pp_department', 'Puppet Node Department Name'],
     ["1.3.6.1.4.1.34380.1.1.16", 'pp_cluster', 'Puppet Node Cluster Name'],
     ["1.3.6.1.4.1.34380.1.1.17", 'pp_provisioner', 'Puppet Node Provisioner Name'],
-    
+    ["1.3.6.1.4.1.34380.1.1.18", 'pp_region', 'Puppet Node Region Name'],
+    ["1.3.6.1.4.1.34380.1.1.19", 'pp_datacenter', 'Puppet Node Datacenter Name'],
+    ["1.3.6.1.4.1.34380.1.1.20", 'pp_zone', 'Puppet Node Zone Name'],
+    ["1.3.6.1.4.1.34380.1.1.21", 'pp_network', 'Puppet Node Network Name'],
+    ["1.3.6.1.4.1.34380.1.1.22", 'pp_securitypolicy', 'Puppet Node Security Policy Name'],
+    ["1.3.6.1.4.1.34380.1.1.23", 'pp_cloudplatform', 'Puppet Node Cloud Platform Name'],
+    ["1.3.6.1.4.1.34380.1.1.24", 'pp_apptier', 'Puppet Node Application Tier'],
+    ["1.3.6.1.4.1.34380.1.1.25", 'pp_hostname', 'Puppet Node Hostname'],
+
     ["1.3.6.1.4.1.34380.1.2", 'ppPrivCertExt', 'Puppet Private Certificate Extension'],
+
+    ["1.3.6.1.4.1.34380.1.3", 'ppAuthCertExt', 'Puppet Certificate Authorization Extension'],
+
+    ["1.3.6.1.4.1.34380.1.3.1",  'pp_authorization', 'Certificate Extension Authorization'],
+    ["1.3.6.1.4.1.34380.1.3.13", 'pp_auth_role', 'Puppet Node Role Name for Authorization'],
   ]
 
   # Register our custom Puppet OIDs with OpenSSL so they can be used as CSR
@@ -61,7 +79,7 @@ module Puppet::SSL::Oids
     end
   end
 
-  # Parse and load custom OID mapping file that enables custom OIDs to be resolved
+  # Parse custom OID mapping file that enables custom OIDs to be resolved
   # into user-friendly names.
   #
   # @param custom_oid_file [String] File to obtain custom OIDs mapping from
@@ -76,7 +94,7 @@ module Puppet::SSL::Oids
   #  '1.3.6.1.4.1.34380.1.2.1.2':
   #    shortname: 'myothershortname'
   #    longname: 'Other Long name'
-  def self.load_custom_oid_file(custom_oid_file, map_key='oid_mapping')
+  def self.parse_custom_oid_file(custom_oid_file, map_key='oid_mapping')
     if File.exists?(custom_oid_file) && File.readable?(custom_oid_file)
       mapping = nil
       begin
@@ -102,6 +120,28 @@ module Puppet::SSL::Oids
         oid_defns << [oid, shortname, longname]
       end
 
+      oid_defns
+    end
+  end
+
+  # Load custom OID mapping file that enables custom OIDs to be resolved
+  # into user-friendly names.
+  #
+  # @param custom_oid_file [String] File to obtain custom OIDs mapping from
+  # @param map_key [String] Hash key in which custom OIDs mapping is stored
+  #
+  # @example Custom OID mapping file
+  # ---
+  # oid_mapping:
+  #  '1.3.6.1.4.1.34380.1.2.1.1':
+  #    shortname : 'myshortname'
+  #    longname  : 'Long name'
+  #  '1.3.6.1.4.1.34380.1.2.1.2':
+  #    shortname: 'myothershortname'
+  #    longname: 'Other Long name'
+  def self.load_custom_oid_file(custom_oid_file, map_key='oid_mapping')
+    oid_defns = parse_custom_oid_file(custom_oid_file, map_key)
+    unless oid_defns.nil?
       begin
         oid_defns.each do |oid_defn|
           OpenSSL::ASN1::ObjectId.register(*oid_defn)

data/lib/puppet/ssl/validator/default_validator.rb

@@ -16,7 +16,7 @@ class Puppet::SSL::Validator::DefaultValidator #< class Puppet::SSL::Validator
   # Creates a new DefaultValidator, optionally with an SSL Configuration and SSL Host.
   #
   # @param ssl_configuration [Puppet::SSL::Configuration] (a default configuration) ssl_configuration the SSL configuration to use
-  # @param ssl_host [Puppet::SSL::Host] (Puppet::SSL::Host.localhost) the SSL host to use
+  # @param ssl_host [Puppet::SSL::Host] The SSL host to use
   #
   # @api private
   #
@@ -25,7 +25,7 @@ class Puppet::SSL::Validator::DefaultValidator #< class Puppet::SSL::Validator
                                         Puppet[:localcacert], {
                                           :ca_auth_file  => Puppet[:ssl_client_ca_auth]
                                         }),
-      ssl_host = Puppet::SSL::Host.localhost)
+      ssl_host = Puppet.lookup(:ssl_host))
 
     reset!
     @ssl_configuration = ssl_configuration
@@ -48,7 +48,7 @@ class Puppet::SSL::Validator::DefaultValidator #< class Puppet::SSL::Validator
   # chain being verified.
   #
   # From the [OpenSSL
-  # documentation](http://www.openssl.org/docs/ssl/SSL_CTX_set_verify.html):
+  # documentation](https://www.openssl.org/docs/ssl/SSL_CTX_set_verify.html):
   # The `verify_callback` function is used to control the behaviour when the
   # SSL_VERIFY_PEER flag is set. It must be supplied by the application and
   # receives two arguments: preverify_ok indicates, whether the verification of

data/lib/puppet/syntax_checkers/base64.rb

@@ -0,0 +1,41 @@
+# A syntax checker for Base64.
+# @api public
+require 'puppet/syntax_checkers'
+require 'base64'
+class Puppet::SyntaxCheckers::Base64 < Puppet::Plugins::SyntaxCheckers::SyntaxChecker
+
+  # Checks the text for BASE64 syntax issues and reports them to the given acceptor.
+  # This checker allows the most relaxed form of Base64, including newlines and missing padding.
+  # It also accept URLsafe input.
+  #
+  # @param text [String] The text to check
+  # @param syntax [String] The syntax identifier in mime style (e.g. 'base64', 'text/xxx+base64')
+  # @param acceptor [#accept] A Diagnostic acceptor
+  # @param source_pos [Puppet::Pops::Adapters::SourcePosAdapter] A source pos adapter with location information
+  # @api public
+  #
+  def check(text, syntax, acceptor, source_pos)
+    raise ArgumentError.new("Base64 syntax checker: the text to check must be a String.") unless text.is_a?(String)
+    raise ArgumentError.new("Base64 syntax checker: the syntax identifier must be a String, e.g. json, data+json") unless syntax.is_a?(String)
+    raise ArgumentError.new("Base64 syntax checker: invalid Acceptor, got: '#{acceptor.class.name}'.") unless acceptor.is_a?(Puppet::Pops::Validation::Acceptor)
+    cleaned_text = text.gsub(/[\r?\n[:blank:]]/, '')
+    begin
+      # Do a strict decode64 on text with all whitespace stripped since the non strict version
+      # simply skips all non base64 characters
+      Base64.strict_decode64(cleaned_text)
+    rescue => e
+      if (cleaned_text.bytes.to_a.size * 8) % 6 != 0
+        msg2 = "padding is not correct"
+      else
+        msg2 = "contains letters outside strict base 64 range (or whitespace)"
+      end
+      msg = "Base64 syntax checker: Cannot parse invalid Base64 string - #{msg2}"
+
+      # TODO: improve the pops API to allow simpler diagnostic creation while still maintaining capabilities
+      # and the issue code. (In this case especially, where there is only a single error message being issued).
+      #
+      issue = Puppet::Pops::Issues::issue(:ILLEGAL_BASE64) { msg }
+      acceptor.accept(Puppet::Pops::Validation::Diagnostic.new(:error, issue, source_pos.locator.file, source_pos, {}))
+    end
+  end
+end

data/lib/puppet/syntax_checkers/json.rb

@@ -4,8 +4,6 @@ require 'puppet/syntax_checkers'
 class Puppet::SyntaxCheckers::Json < Puppet::Plugins::SyntaxCheckers::SyntaxChecker
 
   # Checks the text for JSON syntax issues and reports them to the given acceptor.
-  # This implementation is abstract, it raises {NotImplementedError} since a subclass should have implemented the
-  # method.
   #
   # Error messages from the checker are capped at 100 chars from the source text.
   #

data/lib/puppet/test/test_helper.rb

@@ -38,7 +38,7 @@ module Puppet::Test
     def self.initialize()
       # This meta class instance variable is used as a guard to ensure that
       # before_each, and after_each are only called once. This problem occurs
-      # when there are more than one puppet test infrastructure "orchestrator in us.
+      # when there are more than one puppet test infrastructure orchestrator in use.
       # The use of both puppetabs-spec_helper, and rodjek-rspec_puppet will cause
       # two resets of the puppet environment, and will cause problem rolling back to
       # a known point as there is no way to differentiate where the calls are coming
@@ -84,7 +84,7 @@ module Puppet::Test
     #
     ROLLBACK_MARK = "initial testing state"
 
-    # Call this method once per test, prior to execution of each invididual test.
+    # Call this method once per test, prior to execution of each individual test.
     # @return nil
     def self.before_each_test()
       # When using both rspec-puppet and puppet-rspec-helper, there are two packages trying
@@ -121,7 +121,13 @@ module Puppet::Test
       end
 
       # The process environment is a shared, persistent resource.
-      $old_env = ENV.to_hash
+      # Can't use Puppet.features.microsoft_windows? as it may be mocked out in a test.  This can cause test recurring test failures
+      if (!!File::ALT_SEPARATOR)
+        mode = :windows
+      else
+        mode = :posix
+      end
+      $old_env = Puppet::Util.get_environment(mode)
 
       # So is the load_path
       $old_load_path = $LOAD_PATH.dup
@@ -171,15 +177,24 @@ module Puppet::Test
       end
       $saved_indirection_state = nil
 
+      # Can't use Puppet.features.microsoft_windows? as it may be mocked out in a test.  This can cause test recurring test failures
+      if (!!File::ALT_SEPARATOR)
+        mode = :windows
+      else
+        mode = :posix
+      end
       # Restore the global process environment.  Can't just assign because this
       # is a magic variable, sadly, and doesn't do that™.  It is sufficiently
       # faster to use the compare-then-set model to avoid excessive work that it
       # justifies the complexity.  --daniel 2012-03-15
-      unless ENV.to_hash == $old_env
-        ENV.clear
-        $old_env.each {|k, v| ENV[k] = v }
+      unless Puppet::Util.get_environment(mode) == $old_env
+        Puppet::Util.clear_environment(mode)
+        $old_env.each {|k, v| Puppet::Util.set_env(k, v, mode) }
       end
 
+      # Clear all environments
+      Puppet.lookup(:environments).clear_all
+
       # Restore the load_path late, to avoid messing with stubs from the test.
       $LOAD_PATH.clear
       $old_load_path.each {|x| $LOAD_PATH << x }
@@ -226,7 +241,6 @@ module Puppet::Test
       #
       # I would make these even shorter, but OpenSSL doesn't support anything
       # below 512 bits.  Sad, really, because a 0 bit key would be just fine.
-      Puppet[:req_bits]  = 512
       Puppet[:keylength] = 512
 
       # Although we setup a testing context during initialization, some tests

data/lib/puppet/transaction/additional_resource_generator.rb

@@ -5,10 +5,15 @@
 #
 # @api private
 class Puppet::Transaction::AdditionalResourceGenerator
+  attr_writer :relationship_graph
+  # [boolean] true if any resource has attempted and failed to generate resources
+  attr_reader :resources_failed_to_generate
+
   def initialize(catalog, relationship_graph, prioritizer)
     @catalog = catalog
     @relationship_graph = relationship_graph
     @prioritizer = prioritizer
+    @resources_failed_to_generate = false
   end
 
   def generate_additional_resources(resource)
@@ -16,17 +21,30 @@ class Puppet::Transaction::AdditionalResourceGenerator
     begin
       generated = resource.generate
     rescue => detail
+      @resources_failed_to_generate = true
       resource.log_exception(detail, "Failed to generate additional resources using 'generate': #{detail}")
     end
     return unless generated
     generated = [generated] unless generated.is_a?(Array)
-    generated.collect do |res|
+    generated.collect! do |res|
       @catalog.resource(res.ref) || res
-    end.each do |res|
-      priority = @prioritizer.generate_priority_contained_in(resource, res)
-      add_resource(res, resource, priority)
+    end
+    unless resource.depthfirst?
+      # This is reversed because PUP-1963 changed how generated
+      # resources were added to the catalog. It exists for backwards
+      # compatibility only, and can probably be removed in Puppet 5
+      #
+      # Previously, resources were given sequential priorities in the
+      # relationship graph. Post-1963, resources are added to the
+      # catalog one by one adjacent to the parent resource. This
+      # causes an implicit reversal of their application order from
+      # the old code. The reverse makes it all work like it did.
+      generated.reverse!
+    end
+    generated.each do |res|
+      add_resource(res, resource)
 
-      add_conditional_directed_dependency(resource, res)
+      add_generated_directed_dependency(resource, res)
       generate_additional_resources(res)
     end
   end
@@ -38,6 +56,7 @@ class Puppet::Transaction::AdditionalResourceGenerator
       generated = replace_duplicates_with_catalog_resources(resource.eval_generate)
       return false if generated.empty?
     rescue => detail
+      @resources_failed_to_generate = true
       resource.log_exception(detail, "Failed to generate additional resources using 'eval_generate': #{detail}")
       return false
     end
@@ -60,6 +79,13 @@ class Puppet::Transaction::AdditionalResourceGenerator
 
   def contain_generated_resources_in(resource, made)
     sentinel = Puppet::Type.type(:whit).new(:name => "completed_#{resource.title}", :catalog => resource.catalog)
+    # Tag the completed whit with all of the tags of the generating resource
+    # except the type name to allow event propogation to succeed beyond the whit
+    # "boundary" when filtering resources with tags. We include auto-generated
+    # tags such as the type name to support implicit filtering as well as
+    # explicit. Note that resource#tags returns a duplicate of the resource's
+    # tags.
+    sentinel.tag(*resource.tags)
     priority = @prioritizer.generate_priority_contained_in(resource, sentinel)
     @relationship_graph.add_vertex(sentinel, priority)
 
@@ -102,16 +128,83 @@ class Puppet::Transaction::AdditionalResourceGenerator
     end
   end
 
-  def add_resource(res, parent_resource, priority)
+  def add_resource(res, parent_resource, priority=nil)
     if @catalog.resource(res.ref).nil?
       res.tag(*parent_resource.tags)
-      @catalog.add_resource(res)
-      @relationship_graph.add_vertex(res, priority)
+      if parent_resource.depthfirst?
+        @catalog.add_resource_before(parent_resource, res)
+      else
+        @catalog.add_resource_after(parent_resource, res)
+      end
       @catalog.add_edge(@catalog.container_of(parent_resource), res)
+      if @relationship_graph && priority
+        # If we have a relationship_graph we should add the resource
+        # to it (this is an eval_generate). If we don't, then the
+        # relationship_graph has not yet been created and we can skip
+        # adding it.
+        @relationship_graph.add_vertex(res, priority)
+      end
       res.finish
     end
   end
 
+  # add correct edge for depth- or breadth- first traversal of
+  # generated resource. Skip generating the edge if there is already
+  # some sort of edge between the two resources.
+  def add_generated_directed_dependency(parent, child, label=nil)
+    if parent.depthfirst?
+      source = child
+      target = parent
+    else
+      source = parent
+      target = child
+    end
+
+    # For each potential relationship metaparam, check if parent or
+    # child references the other. If there are none, we should add our
+    # edge.
+    edge_exists = Puppet::Type.relationship_params.any? { |param|
+      param_sym = param.name.to_sym
+
+      if parent[param_sym]
+        parent_contains = parent[param_sym].any? { |res|
+          res.ref == child.ref
+        }
+      else
+        parent_contains = false
+      end
+
+      if child[param_sym]
+        child_contains = child[param_sym].any? { |res|
+          res.ref == parent.ref
+        }
+      else
+        child_contains = false
+      end
+
+      parent_contains || child_contains
+    }
+
+    if not edge_exists
+      # We *cannot* use target.to_resource here!
+      #
+      # For reasons that are beyond my (and, perhaps, human)
+      # comprehension, to_resource will call retrieve. This is
+      # problematic if a generated resource needs the system to be
+      # changed by a previous resource (think a file on a path
+      # controlled by a mount resource).
+      #
+      # Instead of using to_resource, we just construct a resource as
+      # if the arguments to the Type instance had been passed to a
+      # Resource instead.
+      resource = ::Puppet::Resource.new(target.class, target.title,
+                                        :parameters => target.original_parameters)
+
+      source[:before] ||= []
+      source[:before] << resource
+    end
+  end
+
   # Copy an important relationships from the parent to the newly-generated
   # child resource.
   def add_conditional_directed_dependency(parent, child, label=nil)

data/lib/puppet/transaction/event.rb

@@ -11,8 +11,8 @@ class Puppet::Transaction::Event
   include Puppet::Util::Logging
   include Puppet::Network::FormatSupport
 
-  ATTRIBUTES = [:name, :resource, :property, :previous_value, :desired_value, :historical_value, :status, :message, :file, :line, :source_description, :audited, :invalidate_refreshes]
-  YAML_ATTRIBUTES = %w{@audited @property @previous_value @desired_value @historical_value @message @name @status @time}.map(&:to_sym)
+  ATTRIBUTES = [:name, :resource, :property, :previous_value, :desired_value, :historical_value, :status, :message, :file, :line, :source_description, :audited, :invalidate_refreshes, :redacted, :corrective_change]
+  YAML_ATTRIBUTES = %w{@audited @property @previous_value @desired_value @historical_value @message @name @status @time @redacted @corrective_change}.map(&:to_sym)
   attr_accessor *ATTRIBUTES
   attr_accessor :time
   attr_reader :default_log_level
@@ -27,6 +27,8 @@ class Puppet::Transaction::Event
 
   def initialize(options = {})
     @audited = false
+    @redacted = false
+    @corrective_change = false
 
     set_options(options)
     @time = Time.now
@@ -43,6 +45,8 @@ class Puppet::Transaction::Event
     @status = data['status']
     @time = data['time']
     @time = Time.parse(@time) if @time.is_a? String
+    @redacted = data.fetch('redacted', false)
+    @corrective_change = data['corrective_change']
   end
 
   def to_data_hash
@@ -56,10 +60,13 @@ class Puppet::Transaction::Event
       'name' => @name,
       'status' => @status,
       'time' => @time.iso8601(9),
+      'redacted' => @redacted,
+      'corrective_change' => @corrective_change,
     }
   end
 
   def property=(prop)
+    @property_instance = prop
     @property = prop.to_s
   end
 
@@ -91,6 +98,26 @@ class Puppet::Transaction::Event
     YAML_ATTRIBUTES & super
   end
 
+  # Calculate and set the corrective_change parameter, based on the old_system_value of the property.
+  # @param [Object] old_system_value system_value from last transaction
+  # @return [bool] true if this is a corrective_change
+  def calculate_corrective_change(old_system_value)
+    # Only idempotent properties, and cases where we have an old system_value
+    # are corrective_changes.
+    if @property_instance.idempotent? &&
+       !@property_instance.sensitive &&
+       !old_system_value.nil?
+
+      # If the values aren't insync, we have confirmed a corrective_change
+      insync = @property_instance.insync_values?(old_system_value, previous_value)
+
+      # Preserve the nil state, but flip true/false
+      @corrective_change = insync.nil? ? nil : !insync
+    else
+      @corrective_change = false
+    end
+  end
+
   private
 
   # If it's a failure, use 'err', else use either the resource's log level (if available)

data/lib/puppet/transaction/event_manager.rb

@@ -13,7 +13,7 @@ class Puppet::Transaction::EventManager
   # @!attribute [r] events
   #   @todo Determine if this instance variable is used for anything aside from testing.
   #   @return [Array<Puppet::Transaction::Events>] A list of events that can be
-  #     handled by the target resouce. Events that cannot be handled by the
+  #     handled by the target resource. Events that cannot be handled by the
   #     target resource will be discarded.
   attr_reader :events
 

data/lib/puppet/transaction/persistence.rb

@@ -0,0 +1,84 @@
+require 'yaml'
+require 'puppet/util/yaml'
+
+# A persistence store implementation for storing information between
+# transaction runs for the purposes of information inference (such
+# as calculating corrective_change).
+# @api private
+class Puppet::Transaction::Persistence
+  def initialize
+    @old_data = {}
+    @new_data = {"resources" => {}}
+  end
+
+  # Obtain the full raw data from the persistence store.
+  # @return [Hash] hash of data stored in persistence store
+  def data
+    @old_data
+  end
+
+  # Retrieve the system value using the resource and parameter name
+  # @param [String] resource_name name of resource
+  # @param [String] param_name name of the parameter
+  # @return [Object,nil] the system_value
+  def get_system_value(resource_name, param_name)
+    if !@old_data["resources"].nil? &&
+       !@old_data["resources"][resource_name].nil? &&
+       !@old_data["resources"][resource_name]["parameters"].nil? &&
+       !@old_data["resources"][resource_name]["parameters"][param_name].nil?
+      @old_data["resources"][resource_name]["parameters"][param_name]["system_value"]
+    else
+      nil
+    end
+  end
+
+  def set_system_value(resource_name, param_name, value)
+    @new_data["resources"] ||= {}
+    @new_data["resources"][resource_name] ||= {}
+    @new_data["resources"][resource_name]["parameters"] ||= {}
+    @new_data["resources"][resource_name]["parameters"][param_name] ||= {}
+    @new_data["resources"][resource_name]["parameters"][param_name]["system_value"] = value
+  end
+
+  # Load data from the persistence store on disk.
+  def load
+    filename = Puppet[:transactionstorefile]
+    unless Puppet::FileSystem.exist?(filename)
+      return
+    end
+    unless File.file?(filename)
+      Puppet.warning("Transaction store file #{filename} is not a file, ignoring")
+      return
+    end
+
+    result = nil
+    Puppet::Util.benchmark(:debug, "Loaded transaction store file") do
+      begin
+        result = Puppet::Util::Yaml.load_file(filename, false, true)
+      rescue Puppet::Util::Yaml::YamlLoadError => detail
+        Puppet.log_exception(detail, "Transaction store file #{filename} is corrupt (#{detail}); replacing", { :level => :warning })
+
+        begin
+          File.rename(filename, filename + ".bad")
+        rescue => detail
+          Puppet.log_exception(detail, "Unable to rename corrupt transaction store file: #{detail}")
+          raise Puppet::Error, "Could not rename corrupt transaction store file #{filename}; remove manually", detail.backtrace
+        end
+
+        result = {}
+      end
+    end
+
+    unless result.is_a?(Hash)
+      Puppet.err "Transaction store file #{filename} is valid YAML but not returning a hash. Check the file for corruption, or remove it before continuing."
+      return
+    end
+
+    @old_data = result
+  end
+
+  # Save data from internal class to persistence store on disk.
+  def save
+    Puppet::Util::Yaml.dump(@new_data, Puppet[:transactionstorefile])
+  end
+end