puppet 4.4.0-universal-darwin

data/lib/puppet/transaction/report.rb

@@ -0,0 +1,408 @@
+require 'puppet'
+require 'puppet/indirector'
+
+# This class is used to report what happens on a client.
+# There are two types of data in a report; _Logs_ and _Metrics_.
+#
+# * **Logs** - are the output that each change produces.
+# * **Metrics** - are all of the numerical data involved in the transaction.
+#
+# Use {Puppet::Reports} class to create a new custom report type. This class is indirectly used
+# as a source of data to report in such a registered report.
+#
+# ##Metrics
+# There are three types of metrics in each report, and each type of metric has one or more values.
+#
+# * Time: Keeps track of how long things took.
+#   * Total: Total time for the configuration run
+#   * File:
+#   * Exec:
+#   * User:
+#   * Group:
+#   * Config Retrieval: How long the configuration took to retrieve
+#   * Service:
+#   * Package:
+# * Resources: Keeps track of the following stats:
+#   * Total: The total number of resources being managed
+#   * Skipped: How many resources were skipped, because of either tagging or scheduling restrictions
+#   * Scheduled: How many resources met any scheduling restrictions
+#   * Out of Sync: How many resources were out of sync
+#   * Applied: How many resources were attempted to be fixed
+#   * Failed: How many resources were not successfully fixed
+#   * Restarted: How many resources were restarted because their dependencies changed
+#   * Failed Restarts: How many resources could not be restarted
+# * Changes: The total number of changes in the transaction.
+#
+# @api public
+class Puppet::Transaction::Report
+  extend Puppet::Indirector
+
+  indirects :report, :terminus_class => :processor
+
+  # The version of the configuration
+  # @todo Uncertain what this is?
+  # @return [???] the configuration version
+  attr_accessor :configuration_version
+
+  # An agent generated transaction uuid, useful for connecting catalog and report
+  # @return [String] uuid
+  attr_accessor :transaction_uuid
+
+  # The id of the code input to the compiler.
+  attr_accessor :code_id
+
+  # A master generated catalog uuid, useful for connecting a single catalog to multiple reports.
+  attr_accessor :catalog_uuid
+
+  # Whether a cached catalog was used in the run, and if so, the reason that it was used.
+  # @return [String] One of the values: 'not_used', 'explicitly_requested',
+  # or 'on_failure'
+  attr_accessor :cached_catalog_status
+
+  # The host name for which the report is generated
+  # @return [String] the host name
+  attr_accessor :host
+
+  # The name of the environment the host is in
+  # @return [String] the environment name
+  attr_accessor :environment
+
+  # A hash with a map from resource to status
+  # @return [Hash{String => Puppet::Resource::Status}] Resource name to status.
+  attr_reader :resource_statuses
+
+  # A list of log messages.
+  # @return [Array<Puppet::Util::Log>] logged messages
+  attr_reader :logs
+
+  # A hash of metric name to metric value.
+  # @return [Hash<{String => Object}>] A map of metric name to value.
+  # @todo Uncertain if all values are numbers - now marked as Object.
+  #
+  attr_reader :metrics
+
+  # The time when the report data was generated.
+  # @return [Time] A time object indicating when the report data was generated
+  #
+  attr_reader :time
+
+  # The 'kind' of report is the name of operation that triggered the report to be produced.
+  # Typically "apply".
+  # @return [String] the kind of operation that triggered the generation of the report.
+  #
+  attr_reader :kind
+
+  # The status of the client run is an enumeration: 'failed', 'changed' or 'unchanged'
+  # @return [String] the status of the run - one of the values 'failed', 'changed', or 'unchanged'
+  #
+  attr_reader :status
+
+  # @return [String] The Puppet version in String form.
+  # @see Puppet::version()
+  #
+  attr_reader :puppet_version
+
+  # @return [Integer] report format version number.  This value is constant for
+  #    a given version of Puppet; it is incremented when a new release of Puppet
+  #    changes the API for the various objects that make up a report.
+  #
+  attr_reader :report_format
+
+  def self.from_data_hash(data)
+    obj = self.allocate
+    obj.initialize_from_hash(data)
+    obj
+  end
+
+  def as_logging_destination(&block)
+    Puppet::Util::Log.with_destination(self, &block)
+  end
+
+  # @api private
+  def <<(msg)
+    @logs << msg
+    self
+  end
+
+  # @api private
+  def add_times(name, value)
+    @external_times[name] = value
+  end
+
+  # @api private
+  def add_metric(name, hash)
+    metric = Puppet::Util::Metric.new(name)
+
+    hash.each do |metric_name, value|
+      metric.newvalue(metric_name, value)
+    end
+
+    @metrics[metric.name] = metric
+    metric
+  end
+
+  # @api private
+  def add_resource_status(status)
+    @resource_statuses[status.resource] = status
+  end
+
+  # @api private
+  def compute_status(resource_metrics, change_metric)
+    if (resource_metrics["failed"] || 0) > 0
+      'failed'
+    elsif change_metric > 0
+      'changed'
+    else
+      'unchanged'
+    end
+  end
+
+  # @api private
+  def prune_internal_data
+    resource_statuses.delete_if {|name,res| res.resource_type == 'Whit'}
+  end
+
+  # @api private
+  def finalize_report
+    prune_internal_data
+
+    resource_metrics = add_metric(:resources, calculate_resource_metrics)
+    add_metric(:time, calculate_time_metrics)
+    change_metric = calculate_change_metric
+    add_metric(:changes, {"total" => change_metric})
+    add_metric(:events, calculate_event_metrics)
+    @status = compute_status(resource_metrics, change_metric)
+  end
+
+  # @api private
+  def initialize(kind, configuration_version=nil, environment=nil, transaction_uuid=nil)
+    @metrics = {}
+    @logs = []
+    @resource_statuses = {}
+    @external_times ||= {}
+    @host = Puppet[:node_name_value]
+    @time = Time.now
+    @kind = kind
+    @report_format = 5
+    @puppet_version = Puppet.version
+    @configuration_version = configuration_version
+    @transaction_uuid = transaction_uuid
+    @code_id = nil
+    @catalog_uuid = nil
+    @cached_catalog_status = nil
+    @environment = environment
+    @status = 'failed' # assume failed until the report is finalized
+  end
+
+  # @api private
+  def initialize_from_hash(data)
+    @puppet_version = data['puppet_version']
+    @report_format = data['report_format']
+    @configuration_version = data['configuration_version']
+    @transaction_uuid = data['transaction_uuid']
+    @environment = data['environment']
+    @status = data['status']
+    @host = data['host']
+    @time = data['time']
+
+    if catalog_uuid = data['catalog_uuid']
+      @catalog_uuid = catalog_uuid
+    end
+
+    if code_id = data['code_id']
+      @code_id = code_id
+    end
+
+    if cached_catalog_status = data['cached_catalog_status']
+      @cached_catalog_status = cached_catalog_status
+    end
+
+    if @time.is_a? String
+      @time = Time.parse(@time)
+    end
+    @kind = data['kind']
+
+    @metrics = {}
+    data['metrics'].each do |name, hash|
+      @metrics[name] = Puppet::Util::Metric.from_data_hash(hash)
+    end
+
+    @logs = data['logs'].map do |record|
+      Puppet::Util::Log.from_data_hash(record)
+    end
+
+    @resource_statuses = {}
+    data['resource_statuses'].map do |record|
+      if record[1] == {}
+        status = nil
+      else
+        status = Puppet::Resource::Status.from_data_hash(record[1])
+      end
+      @resource_statuses[record[0]] = status
+    end
+  end
+
+  def to_data_hash
+    {
+      'host' => @host,
+      'time' => @time.iso8601(9),
+      'configuration_version' => @configuration_version,
+      'transaction_uuid' => @transaction_uuid,
+      'catalog_uuid' => @catalog_uuid,
+      'code_id' => @code_id,
+      'cached_catalog_status' => @cached_catalog_status,
+      'report_format' => @report_format,
+      'puppet_version' => @puppet_version,
+      'kind' => @kind,
+      'status' => @status,
+      'environment' => @environment,
+
+      'logs' => @logs,
+      'metrics' => @metrics,
+      'resource_statuses' => @resource_statuses,
+    }
+  end
+
+  # @return [String] the host name
+  # @api public
+  #
+  def name
+    host
+  end
+
+  # Provide a human readable textual summary of this report.
+  # @note This is intended for debugging purposes
+  # @return [String] A string with a textual summary of this report.
+  # @api public
+  #
+  def summary
+    report = raw_summary
+
+    ret = ""
+    report.keys.sort { |a,b| a.to_s <=> b.to_s }.each do |key|
+      ret += "#{Puppet::Util::Metric.labelize(key)}:\n"
+
+      report[key].keys.sort { |a,b|
+        # sort by label
+        if a == :total
+          1
+        elsif b == :total
+          -1
+        else
+          report[key][a].to_s <=> report[key][b].to_s
+        end
+      }.each do |label|
+        value = report[key][label]
+        next if value == 0
+        value = "%0.2f" % value if value.is_a?(Float)
+        ret += "   %15s %s\n" % [Puppet::Util::Metric.labelize(label) + ":", value]
+      end
+    end
+    ret
+  end
+
+  # Provides a raw hash summary of this report.
+  # @return [Hash<{String => Object}>] A hash with metrics key to value map
+  # @api public
+  #
+  def raw_summary
+    report = { "version" => { "config" => configuration_version, "puppet" => Puppet.version  } }
+
+    @metrics.each do |name, metric|
+      key = metric.name.to_s
+      report[key] = {}
+      metric.values.each do |metric_name, label, value|
+        report[key][metric_name.to_s] = value
+      end
+      report[key]["total"] = 0 unless key == "time" or report[key].include?("total")
+    end
+    (report["time"] ||= {})["last_run"] = Time.now.tv_sec
+    report
+  end
+
+  # Computes a single number that represents the report's status.
+  # The computation is based on the contents of this report's metrics.
+  # The resulting number is a bitmask where
+  # individual bits represent the presence of different metrics.
+  #
+  # * 0x2 set if there are changes
+  # * 0x4 set if there are resource failures or resources that failed to restart
+  # @return [Integer] A bitmask where 0x2 is set if there are changes, and 0x4 is set of there are failures.
+  # @api public
+  #
+  def exit_status
+    status = 0
+    status |= 2 if @metrics["changes"]["total"] > 0
+    status |= 4 if @metrics["resources"]["failed"] > 0
+    status |= 4 if @metrics["resources"]["failed_to_restart"] > 0
+    status
+  end
+
+  # @api private
+  #
+  def to_yaml_properties
+    super - [:@external_times]
+  end
+
+  def self.supported_formats
+    [:pson, :yaml]
+  end
+
+  def self.default_format
+    :pson
+  end
+
+  private
+
+  def calculate_change_metric
+    resource_statuses.map { |name, status| status.change_count || 0 }.inject(0) { |a,b| a+b }
+  end
+
+  def calculate_event_metrics
+    metrics = Hash.new(0)
+    %w{total failure success}.each { |m| metrics[m] = 0 }
+    resource_statuses.each do |name, status|
+      metrics["total"] += status.events.length
+      status.events.each do |event|
+        metrics[event.status] += 1
+      end
+    end
+
+    metrics
+  end
+
+  def calculate_resource_metrics
+    metrics = {}
+    metrics["total"] = resource_statuses.length
+
+    # force every resource key in the report to be present
+    # even if no resources is in this given state
+    Puppet::Resource::Status::STATES.each do |state|
+      metrics[state.to_s] = 0
+    end
+
+    resource_statuses.each do |name, status|
+      Puppet::Resource::Status::STATES.each do |state|
+        metrics[state.to_s] += 1 if status.send(state)
+      end
+    end
+
+    metrics
+  end
+
+  def calculate_time_metrics
+    metrics = Hash.new(0)
+    resource_statuses.each do |name, status|
+      type = Puppet::Resource.new(name).type
+      metrics[type.to_s.downcase] += status.evaluation_time if status.evaluation_time
+    end
+
+    @external_times.each do |name, value|
+      metrics[name.to_s.downcase] = value
+    end
+
+    metrics["total"] = metrics.values.inject(0) { |a,b| a+b }
+
+    metrics
+  end
+end

data/lib/puppet/transaction/resource_harness.rb

@@ -0,0 +1,236 @@
+require 'puppet/resource/status'
+
+class Puppet::Transaction::ResourceHarness
+  NO_ACTION = Object.new
+
+  extend Forwardable
+  def_delegators :@transaction, :relationship_graph
+
+  attr_reader :transaction
+
+  def initialize(transaction)
+    @transaction = transaction
+  end
+
+  def evaluate(resource)
+    status = Puppet::Resource::Status.new(resource)
+
+    begin
+      context = ResourceApplicationContext.from_resource(resource, status)
+      perform_changes(resource, context)
+
+      if status.changed? && ! resource.noop?
+        cache(resource, :synced, Time.now)
+        resource.flush if resource.respond_to?(:flush)
+      end
+    rescue => detail
+      status.failed_because(detail)
+    ensure
+      status.evaluation_time = Time.now - status.time
+    end
+
+    status
+  end
+
+  def scheduled?(resource)
+    return true if Puppet[:ignoreschedules]
+    return true unless schedule = schedule(resource)
+
+    # We use 'checked' here instead of 'synced' because otherwise we'll
+    # end up checking most resources most times, because they will generally
+    # have been synced a long time ago (e.g., a file only gets updated
+    # once a month on the server and its schedule is daily; the last sync time
+    # will have been a month ago, so we'd end up checking every run).
+    schedule.match?(cached(resource, :checked).to_i)
+  end
+
+  def schedule(resource)
+    unless resource.catalog
+      resource.warning "Cannot schedule without a schedule-containing catalog"
+      return nil
+    end
+
+    return nil unless name = resource[:schedule]
+    resource.catalog.resource(:schedule, name) || resource.fail("Could not find schedule #{name}")
+  end
+
+  # Used mostly for scheduling and auditing at this point.
+  def cached(resource, name)
+    Puppet::Util::Storage.cache(resource)[name]
+  end
+
+  # Used mostly for scheduling and auditing at this point.
+  def cache(resource, name, value)
+    Puppet::Util::Storage.cache(resource)[name] = value
+  end
+
+  private
+
+  def perform_changes(resource, context)
+    cache(resource, :checked, Time.now)
+
+    # Record the current state in state.yml.
+    context.audited_params.each do |param|
+      cache(resource, param, context.current_values[param])
+    end
+
+    ensure_param = resource.parameter(:ensure)
+    if ensure_param && ensure_param.should
+      ensure_event = sync_if_needed(ensure_param, context)
+    else
+      ensure_event = NO_ACTION
+    end
+
+    if ensure_event == NO_ACTION
+      if context.resource_present?
+        resource.properties.each do |param|
+          sync_if_needed(param, context)
+        end
+      else
+        resource.debug("Nothing to manage: no ensure and the resource doesn't exist")
+      end
+    end
+
+    capture_audit_events(resource, context)
+  end
+
+  def sync_if_needed(param, context)
+    historical_value = context.historical_values[param.name]
+    current_value = context.current_values[param.name]
+    do_audit = context.audited_params.include?(param.name)
+
+    begin
+      if param.should && !param.safe_insync?(current_value)
+        event = create_change_event(param, current_value, historical_value)
+        if do_audit
+          event = audit_event(event, param)
+        end
+
+        brief_audit_message = audit_message(param, do_audit, historical_value, current_value)
+
+        if param.noop
+          noop(event, param, current_value, brief_audit_message)
+        else
+          sync(event, param, current_value, brief_audit_message)
+        end
+
+        event
+      else
+        NO_ACTION
+      end
+    rescue => detail
+      # Execution will continue on StandardErrors, just store the event
+      Puppet.log_exception(detail)
+
+      event = create_change_event(param, current_value, historical_value)
+      event.status = "failure"
+      event.message = "change from #{param.is_to_s(current_value)} to #{param.should_to_s(param.should)} failed: #{detail}"
+      event
+    rescue Exception => detail
+      # Execution will halt on Exceptions, they get raised to the application
+      event = create_change_event(param, current_value, historical_value)
+      event.status = "failure"
+      event.message = "change from #{param.is_to_s(current_value)} to #{param.should_to_s(param.should)} failed: #{detail}"
+      raise
+    ensure
+      if event
+        context.record(event)
+        event.send_log
+        context.synced_params << param.name
+      end
+    end
+  end
+
+  def create_change_event(property, current_value, historical_value)
+    event = property.event
+    event.previous_value = current_value
+    event.desired_value = property.should
+    event.historical_value = historical_value
+
+    event
+  end
+
+  # This method is an ugly hack because, given a Time object with nanosecond
+  # resolution, roundtripped through YAML serialization, the Time object will
+  # be truncated to microseconds.
+  # For audit purposes, this code special cases this comparison, and compares
+  # the two objects by their second and microsecond components. tv_sec is the
+  # number of seconds since the epoch, and tv_usec is only the microsecond
+  # portion of time.
+  def are_audited_values_equal(a, b)
+    a == b || (a.is_a?(Time) && b.is_a?(Time) && a.tv_sec == b.tv_sec && a.tv_usec == b.tv_usec)
+  end
+  private :are_audited_values_equal
+
+  def audit_event(event, property)
+    event.audited = true
+    event.status = "audit"
+    if !are_audited_values_equal(event.historical_value, event.previous_value)
+      event.message = "audit change: previously recorded value #{property.is_to_s(event.historical_value)} has been changed to #{property.is_to_s(event.previous_value)}"
+    end
+
+    event
+  end
+
+  def audit_message(param, do_audit, historical_value, current_value)
+    if do_audit && historical_value && !are_audited_values_equal(historical_value, current_value)
+      " (previously recorded value was #{param.is_to_s(historical_value)})"
+    else
+      ""
+    end
+  end
+
+  def noop(event, param, current_value, audit_message)
+    event.message = "current_value #{param.is_to_s(current_value)}, should be #{param.should_to_s(param.should)} (noop)#{audit_message}"
+    event.status = "noop"
+  end
+
+  def sync(event, param, current_value, audit_message)
+    param.sync
+    event.message = "#{param.change_to_s(current_value, param.should)}#{audit_message}"
+    event.status = "success"
+  end
+
+  def capture_audit_events(resource, context)
+    context.audited_params.each do |param_name|
+      if context.historical_values.include?(param_name)
+        if !are_audited_values_equal(context.historical_values[param_name], context.current_values[param_name]) && !context.synced_params.include?(param_name)
+          parameter = resource.parameter(param_name)
+          event = audit_event(create_change_event(parameter,
+                                                  context.current_values[param_name],
+                                                  context.historical_values[param_name]),
+                              parameter)
+          event.send_log
+          context.record(event)
+        end
+      else
+        resource.property(param_name).notice "audit change: newly-recorded value #{context.current_values[param_name]}"
+      end
+    end
+  end
+
+  # @api private
+  ResourceApplicationContext = Struct.new(:resource,
+                                          :current_values,
+                                          :historical_values,
+                                          :audited_params,
+                                          :synced_params,
+                                          :status) do
+    def self.from_resource(resource, status)
+      ResourceApplicationContext.new(resource,
+                                     resource.retrieve_resource.to_hash,
+                                     Puppet::Util::Storage.cache(resource).dup,
+                                     (resource[:audit] || []).map { |p| p.to_sym },
+                                     [],
+                                     status)
+    end
+
+    def resource_present?
+      resource.present?(current_values)
+    end
+
+    def record(event)
+      status << event
+    end
+  end
+end