puppet 4.5.3-universal-darwin → 4.6.1-universal-darwin

data/lib/puppet/transaction.rb

@@ -15,6 +15,7 @@ class Puppet::Transaction
   require 'puppet/transaction/event_manager'
   require 'puppet/transaction/resource_harness'
   require 'puppet/resource/status'
+  require 'puppet/transaction/persistence'
 
   attr_accessor :catalog, :ignoreschedules, :for_network_device
 
@@ -29,12 +30,19 @@ class Puppet::Transaction
 
   attr_reader :prefetched_providers
 
+  # @!attribute [r] persistence
+  #   @return [Puppet::Transaction::Persistence] persistence object for cross
+  #      transaction storage.
+  attr_reader :persistence
+
   include Puppet::Util
   include Puppet::Util::Tagging
 
   def initialize(catalog, report, prioritizer)
     @catalog = catalog
 
+    @persistence = Puppet::Transaction::Persistence.new
+
     @report = report || Puppet::Transaction::Report.new("apply", catalog.version, catalog.environment)
 
     @prioritizer = prioritizer
@@ -85,6 +93,8 @@ class Puppet::Transaction
 
     perform_pre_run_checks
 
+    persistence.load if catalog.host_config?
+
     Puppet.info "Applying configuration version '#{catalog.version}'" if catalog.version
 
     continue_while = lambda { !stop_processing? }
@@ -134,6 +144,8 @@ class Puppet::Transaction
           Puppet.log_exception(detail, "post_resource_eval failed for provider #{provider}")
         end
       end
+
+      persistence.save if catalog.host_config?
     end
 
     # Generate the relationship graph, set up our generator to use it

data/lib/puppet/transaction/additional_resource_generator.rb

@@ -26,7 +26,7 @@ class Puppet::Transaction::AdditionalResourceGenerator
       @catalog.resource(res.ref) || res
     end
     unless resource.depthfirst?
-      # This is reversed becuase PUP-1963 changed how generated
+      # This is reversed because PUP-1963 changed how generated
       # resources were added to the catalog. It exists for backwards
       # compatibility only, and can probably be removed in Puppet 5
       #

data/lib/puppet/transaction/event.rb

@@ -11,8 +11,8 @@ class Puppet::Transaction::Event
   include Puppet::Util::Logging
   include Puppet::Network::FormatSupport
 
-  ATTRIBUTES = [:name, :resource, :property, :previous_value, :desired_value, :historical_value, :status, :message, :file, :line, :source_description, :audited, :invalidate_refreshes]
-  YAML_ATTRIBUTES = %w{@audited @property @previous_value @desired_value @historical_value @message @name @status @time}.map(&:to_sym)
+  ATTRIBUTES = [:name, :resource, :property, :previous_value, :desired_value, :historical_value, :status, :message, :file, :line, :source_description, :audited, :invalidate_refreshes, :redacted, :corrective_change]
+  YAML_ATTRIBUTES = %w{@audited @property @previous_value @desired_value @historical_value @message @name @status @time @redacted @corrective_change}.map(&:to_sym)
   attr_accessor *ATTRIBUTES
   attr_accessor :time
   attr_reader :default_log_level
@@ -27,6 +27,8 @@ class Puppet::Transaction::Event
 
   def initialize(options = {})
     @audited = false
+    @redacted = false
+    @corrective_change = false
 
     set_options(options)
     @time = Time.now
@@ -43,6 +45,8 @@ class Puppet::Transaction::Event
     @status = data['status']
     @time = data['time']
     @time = Time.parse(@time) if @time.is_a? String
+    @redacted = data.fetch('redacted', false)
+    @corrective_change = data['corrective_change']
   end
 
   def to_data_hash
@@ -56,10 +60,13 @@ class Puppet::Transaction::Event
       'name' => @name,
       'status' => @status,
       'time' => @time.iso8601(9),
+      'redacted' => @redacted,
+      'corrective_change' => @corrective_change,
     }
   end
 
   def property=(prop)
+    @property_instance = prop
     @property = prop.to_s
   end
 
@@ -91,6 +98,26 @@ class Puppet::Transaction::Event
     YAML_ATTRIBUTES & super
   end
 
+  # Calculate and set the corrective_change parameter, based on the old_system_value of the property.
+  # @param [Object] old_system_value system_value from last transaction
+  # @return [bool] true if this is a corrective_change
+  def calculate_corrective_change(old_system_value)
+    # Only idempotent properties, and cases where we have an old system_value
+    # are corrective_changes.
+    if @property_instance.idempotent? &&
+       !@property_instance.sensitive &&
+       !old_system_value.nil?
+
+      # If the values aren't insync, we have confirmed a corrective_change
+      insync = @property_instance.insync_values?(old_system_value, previous_value)
+
+      # Preserve the nil state, but flip true/false
+      @corrective_change = insync.nil? ? nil : !insync
+    else
+      @corrective_change = false
+    end
+  end
+
   private
 
   # If it's a failure, use 'err', else use either the resource's log level (if available)

data/lib/puppet/transaction/event_manager.rb

@@ -13,7 +13,7 @@ class Puppet::Transaction::EventManager
   # @!attribute [r] events
   #   @todo Determine if this instance variable is used for anything aside from testing.
   #   @return [Array<Puppet::Transaction::Events>] A list of events that can be
-  #     handled by the target resouce. Events that cannot be handled by the
+  #     handled by the target resource. Events that cannot be handled by the
   #     target resource will be discarded.
   attr_reader :events
 

data/lib/puppet/transaction/persistence.rb

@@ -0,0 +1,84 @@
+require 'yaml'
+require 'puppet/util/yaml'
+
+# A persistence store implementation for storing information between
+# transaction runs for the purposes of information inference (such
+# as calculating corrective_change).
+# @api private
+class Puppet::Transaction::Persistence
+  def initialize
+    @old_data = {}
+    @new_data = {"resources" => {}}
+  end
+
+  # Obtain the full raw data from the persistence store.
+  # @return [Hash] hash of data stored in persistence store
+  def data
+    @old_data
+  end
+
+  # Retrieve the system value using the resource and parameter name
+  # @param [String] resource_name name of resource
+  # @param [String] param_name name of the parameter
+  # @return [Object,nil] the system_value
+  def get_system_value(resource_name, param_name)
+    if !@old_data["resources"].nil? &&
+       !@old_data["resources"][resource_name].nil? &&
+       !@old_data["resources"][resource_name]["parameters"].nil? &&
+       !@old_data["resources"][resource_name]["parameters"][param_name].nil?
+      @old_data["resources"][resource_name]["parameters"][param_name]["system_value"]
+    else
+      nil
+    end
+  end
+
+  def set_system_value(resource_name, param_name, value)
+    @new_data["resources"] ||= {}
+    @new_data["resources"][resource_name] ||= {}
+    @new_data["resources"][resource_name]["parameters"] ||= {}
+    @new_data["resources"][resource_name]["parameters"][param_name] ||= {}
+    @new_data["resources"][resource_name]["parameters"][param_name]["system_value"] = value
+  end
+
+  # Load data from the persistence store on disk.
+  def load
+    filename = Puppet[:transactionstorefile]
+    unless Puppet::FileSystem.exist?(filename)
+      return
+    end
+    unless File.file?(filename)
+      Puppet.warning("Transaction store file #{filename} is not a file, ignoring")
+      return
+    end
+
+    result = nil
+    Puppet::Util.benchmark(:debug, "Loaded transaction store file") do
+      begin
+        result = Puppet::Util::Yaml.load_file(filename, false, true)
+      rescue Puppet::Util::Yaml::YamlLoadError => detail
+        Puppet.log_exception(detail, "Transaction store file #{filename} is corrupt (#{detail}); replacing", { :level => :warning })
+
+        begin
+          File.rename(filename, filename + ".bad")
+        rescue => detail
+          Puppet.log_exception(detail, "Unable to rename corrupt transaction store file: #{detail}")
+          raise Puppet::Error, "Could not rename corrupt transaction store file #{filename}; remove manually", detail.backtrace
+        end
+
+        result = {}
+      end
+    end
+
+    unless result.is_a?(Hash)
+      Puppet.err "Transaction store file #{filename} is valid YAML but not returning a hash. Check the file for corruption, or remove it before continuing."
+      return
+    end
+
+    @old_data = result
+  end
+
+  # Save data from internal class to persistence store on disk.
+  def save
+    Puppet::Util::Yaml.dump(@new_data, Puppet[:transactionstorefile])
+  end
+end

data/lib/puppet/transaction/report.rb

@@ -59,6 +59,10 @@ class Puppet::Transaction::Report
   # or 'on_failure'
   attr_accessor :cached_catalog_status
 
+  # Contains the name and port of the master that was successfully contacted
+  # @return [String] a string of the format 'servername:port'
+  attr_accessor :master_used
+
   # The host name for which the report is generated
   # @return [String] the host name
   attr_accessor :host
@@ -67,6 +71,11 @@ class Puppet::Transaction::Report
   # @return [String] the environment name
   attr_accessor :environment
 
+  # Whether there are changes that we decided not to apply because of noop
+  # @return [Boolean]
+  #
+  attr_accessor :noop_pending
+
   # A hash with a map from resource to status
   # @return [Hash{String => Puppet::Resource::Status}] Resource name to status.
   attr_reader :resource_statuses
@@ -108,6 +117,16 @@ class Puppet::Transaction::Report
   #
   attr_reader :report_format
 
+  # Whether the puppet run was started in noop mode
+  # @return [Boolean]
+  #
+  attr_reader :noop
+
+  # @!attribute [r] corrective_change
+  #   @return [Boolean] true if the report contains any events and resources that had
+  #      corrective changes.
+  attr_reader :corrective_change
+
   def self.from_data_hash(data)
     obj = self.allocate
     obj.initialize_from_hash(data)
@@ -157,6 +176,11 @@ class Puppet::Transaction::Report
     end
   end
 
+  # @api private
+  def has_noop_events?(resource)
+    resource.events.any? { |event| event.status == 'noop' }
+  end
+
   # @api private
   def prune_internal_data
     resource_statuses.delete_if {|name,res| res.resource_type == 'Whit'}
@@ -165,6 +189,7 @@ class Puppet::Transaction::Report
   # @api private
   def finalize_report
     prune_internal_data
+    calculate_report_corrective_change
 
     resource_metrics = add_metric(:resources, calculate_resource_metrics)
     add_metric(:time, calculate_time_metrics)
@@ -172,6 +197,7 @@ class Puppet::Transaction::Report
     add_metric(:changes, {"total" => change_metric})
     add_metric(:events, calculate_event_metrics)
     @status = compute_status(resource_metrics, change_metric)
+    @noop_pending = @resource_statuses.any? { |name,res| has_noop_events?(res) }
   end
 
   # @api private
@@ -183,15 +209,19 @@ class Puppet::Transaction::Report
     @host = Puppet[:node_name_value]
     @time = Time.now
     @kind = kind
-    @report_format = 5
+    @report_format = 6
     @puppet_version = Puppet.version
     @configuration_version = configuration_version
     @transaction_uuid = transaction_uuid
     @code_id = nil
     @catalog_uuid = nil
     @cached_catalog_status = nil
+    @master_used = nil
     @environment = environment
     @status = 'failed' # assume failed until the report is finalized
+    @noop = Puppet[:noop]
+    @noop_pending = false
+    @corrective_change = false
   end
 
   # @api private
@@ -202,8 +232,15 @@ class Puppet::Transaction::Report
     @transaction_uuid = data['transaction_uuid']
     @environment = data['environment']
     @status = data['status']
+    @noop = data['noop']
+    @noop_pending = data['noop_pending']
     @host = data['host']
     @time = data['time']
+    @corrective_change = data['corrective_change']
+
+    if master_used = data['master_used']
+      @master_used = master_used
+    end
 
     if catalog_uuid = data['catalog_uuid']
       @catalog_uuid = catalog_uuid
@@ -255,11 +292,14 @@ class Puppet::Transaction::Report
       'puppet_version' => @puppet_version,
       'kind' => @kind,
       'status' => @status,
+      'noop' => @noop,
+      'noop_pending' => @noop_pending,
       'environment' => @environment,
-
+      'master_used' => @master_used,
       'logs' => @logs,
       'metrics' => @metrics,
       'resource_statuses' => @resource_statuses,
+      'corrective_change' => @corrective_change,
     }
   end
 
@@ -354,6 +394,13 @@ class Puppet::Transaction::Report
 
   private
 
+  # Mark the report as corrective, if there are any resource_status marked corrective.
+  def calculate_report_corrective_change
+    @corrective_change = resource_statuses.any? do |name, status|
+      status.corrective_change
+    end
+  end
+
   def calculate_change_metric
     resource_statuses.map { |name, status| status.change_count || 0 }.inject(0) { |a,b| a+b }
   end

data/lib/puppet/transaction/resource_harness.rb

@@ -10,6 +10,7 @@ class Puppet::Transaction::ResourceHarness
 
   def initialize(transaction)
     @transaction = transaction
+    @persistence = transaction.persistence
   end
 
   def evaluate(resource)
@@ -92,6 +93,25 @@ class Puppet::Transaction::ResourceHarness
     end
 
     capture_audit_events(resource, context)
+    persist_system_values(resource, context)
+  end
+
+  # We persist the last known values for the properties of a resource after resource
+  # application.
+  # @param [Puppet::Type] resource resource whose values we are to persist.
+  # @param [ResourceApplicationContent] context the application context to operate on.
+  def persist_system_values(resource, context)
+    param_to_event = {}
+    context.status.events.each do |ev|
+      param_to_event[ev.property] = ev
+    end
+
+    context.system_value_params.each do |pname, param|
+      @persistence.set_system_value(resource.ref, pname.to_s,
+                                    new_system_value(param,
+                                                     param_to_event[pname.to_s],
+                                                     @persistence.get_system_value(resource.ref, pname.to_s)))
+    end
   end
 
   def sync_if_needed(param, context)
@@ -103,7 +123,7 @@ class Puppet::Transaction::ResourceHarness
       if param.should && !param.safe_insync?(current_value)
         event = create_change_event(param, current_value, historical_value)
         if do_audit
-          event = audit_event(event, param)
+          event = audit_event(event, param, context)
         end
 
         brief_audit_message = audit_message(param, do_audit, historical_value, current_value)
@@ -124,16 +144,21 @@ class Puppet::Transaction::ResourceHarness
 
       event = create_change_event(param, current_value, historical_value)
       event.status = "failure"
-      event.message = "change from #{param.is_to_s(current_value)} to #{param.should_to_s(param.should)} failed: #{detail}"
+      event.message = param.format("change from %s to %s failed: #{detail}",
+                                   param.is_to_s(current_value),
+                                   param.should_to_s(param.should))
       event
     rescue Exception => detail
       # Execution will halt on Exceptions, they get raised to the application
       event = create_change_event(param, current_value, historical_value)
       event.status = "failure"
-      event.message = "change from #{param.is_to_s(current_value)} to #{param.should_to_s(param.should)} failed: #{detail}"
+      event.message = param.format("change from %s to %s failed: #{detail}",
+                                   param.is_to_s(current_value),
+                                   param.should_to_s(param.should))
       raise
     ensure
       if event
+        event.calculate_corrective_change(@persistence.get_system_value(context.resource.ref, param.name.to_s))
         context.record(event)
         event.send_log
         context.synced_params << param.name
@@ -142,12 +167,20 @@ class Puppet::Transaction::ResourceHarness
   end
 
   def create_change_event(property, current_value, historical_value)
-    event = property.event
-    event.previous_value = current_value
-    event.desired_value = property.should
-    event.historical_value = historical_value
+    options = {}
+    should = property.should
 
-    event
+    if property.sensitive
+      options[:previous_value] = current_value.nil? ? nil : '[redacted]'
+      options[:desired_value] = should.nil? ? nil : '[redacted]'
+      options[:historical_value] = historical_value.nil? ? nil : '[redacted]'
+    else
+      options[:previous_value] = current_value
+      options[:desired_value] = should
+      options[:historical_value] = historical_value
+    end
+
+    property.event(options)
   end
 
   # This method is an ugly hack because, given a Time object with nanosecond
@@ -162,11 +195,23 @@ class Puppet::Transaction::ResourceHarness
   end
   private :are_audited_values_equal
 
-  def audit_event(event, property)
+  # Populate an existing event with audit information.
+  #
+  # @param event [Puppet::Transaction::Event] The event to be populated.
+  # @param property [Puppet::Property] The property being audited.
+  # @param context [ResourceApplicationContext]
+  #
+  # @return [Puppet::Transaction::Event] The given event, populated with the audit information.
+  def audit_event(event, property, context)
     event.audited = true
     event.status = "audit"
-    if !are_audited_values_equal(event.historical_value, event.previous_value)
-      event.message = "audit change: previously recorded value #{property.is_to_s(event.historical_value)} has been changed to #{property.is_to_s(event.previous_value)}"
+
+    # The event we've been provided might have been redacted so we need to use the state stored within
+    # the resource application context to see if an event was actually generated.
+    if !are_audited_values_equal(context.historical_values[property.name], context.current_values[property.name])
+      event.message = property.format("audit change: previously recorded value %s has been changed to %s",
+                 property.is_to_s(event.historical_value),
+                 property.is_to_s(event.previous_value))
     end
 
     event
@@ -174,20 +219,26 @@ class Puppet::Transaction::ResourceHarness
 
   def audit_message(param, do_audit, historical_value, current_value)
     if do_audit && historical_value && !are_audited_values_equal(historical_value, current_value)
-      " (previously recorded value was #{param.is_to_s(historical_value)})"
+      param.format(" (previously recorded value was %s)", param.is_to_s(historical_value))
     else
       ""
     end
   end
 
   def noop(event, param, current_value, audit_message)
-    event.message = "current_value #{param.is_to_s(current_value)}, should be #{param.should_to_s(param.should)} (noop)#{audit_message}"
+    event.message = param.format("current_value %s, should be %s (noop)#{audit_message}",
+                                 param.is_to_s(current_value),
+                                 param.should_to_s(param.should))
     event.status = "noop"
   end
 
   def sync(event, param, current_value, audit_message)
     param.sync
-    event.message = "#{param.change_to_s(current_value, param.should)}#{audit_message}"
+    if param.sensitive
+      event.message = param.format("changed %s to %s#{audit_message}", param.is_to_s(current_value), param.should_to_s(param.should))
+    else
+      event.message = "#{param.change_to_s(current_value, param.should)}#{audit_message}"
+    end
     event.status = "success"
   end
 
@@ -199,30 +250,59 @@ class Puppet::Transaction::ResourceHarness
           event = audit_event(create_change_event(parameter,
                                                   context.current_values[param_name],
                                                   context.historical_values[param_name]),
-                              parameter)
+                              parameter, context)
           event.send_log
           context.record(event)
         end
       else
-        resource.property(param_name).notice "audit change: newly-recorded value #{context.current_values[param_name]}"
+        property = resource.property(param_name)
+        property.notice(property.format("audit change: newly-recorded value %s", context.current_values[param_name]))
       end
     end
   end
 
+  # Given an event and its property, calculate the system_value to persist
+  # for future calculations.
+  # @param [Puppet::Transaction::Event] event event to use for processing
+  # @param [Puppet::Property] property correlating property
+  # @param [Object] old_system_value system_value from last transaction
+  # @return [Object] system_value to be used for next transaction
+  def new_system_value(property, event, old_system_value)
+    if event && event.status != "success"
+      # For non-success events, we persist the old_system_value if it is defined,
+      # or use the event previous_value.
+      # If we're using the event previous_value, we ensure that it's
+      # an array. This is needed because properties assume that their
+      # `should` value is an array, and we will use this value later
+      # on in property insync? logic.
+      event_value = [event.previous_value] unless event.previous_value.is_a?(Array)
+      old_system_value.nil? ? event_value : old_system_value
+    else
+      # For non events, or for success cases, we just want to store
+      # the parameters agent value.
+      # We use instance_variable_get here because we want this process to bypass any
+      # munging/unmunging or validation that the property might try to do, since those
+      # operations may not be correctly implemented for custom types.
+      property.instance_variable_get(:@should)
+    end
+  end
+
   # @api private
   ResourceApplicationContext = Struct.new(:resource,
                                           :current_values,
                                           :historical_values,
                                           :audited_params,
                                           :synced_params,
-                                          :status) do
+                                          :status,
+                                          :system_value_params) do
     def self.from_resource(resource, status)
       ResourceApplicationContext.new(resource,
                                      resource.retrieve_resource.to_hash,
                                      Puppet::Util::Storage.cache(resource).dup,
                                      (resource[:audit] || []).map { |p| p.to_sym },
                                      [],
-                                     status)
+                                     status,
+                                     resource.parameters.select { |n,p| p.is_a?(Puppet::Property) && !p.sensitive })
     end
 
     def resource_present?