ml-puppetdb-terminus 3.2.1

data/puppet/lib/puppet/indirector/facts/puppetdb_apply.rb

@@ -0,0 +1,25 @@
+require 'puppet/indirector/facts/puppetdb'
+
+# This class provides an alternative implementation of the Facts::Puppetdb
+# terminus that better suits execution via `puppet apply`.
+#
+# This terminus is designed to be used as a cache terminus, to ensure that facts
+# are stored in PuppetDB. It does not act as a real cache itself however, it
+# tells Puppet to fallback to the `terminus` instead.
+class Puppet::Node::Facts::PuppetdbApply < Puppet::Node::Facts::Puppetdb
+  attr_writer :dbstored
+
+  # Here we override the normal save, only saving the first time, as a `save`
+  # can be called multiple times in a puppet run.
+  def save(args)
+    unless @dbstored
+      @dbstored = true
+      super(args)
+    end
+  end
+
+  # By returning nil, we force puppet to use the real terminus.
+  def find(args)
+    nil
+  end
+end

data/puppet/lib/puppet/indirector/node/puppetdb.rb

@@ -0,0 +1,19 @@
+require 'puppet/node'
+require 'puppet/indirector/rest'
+require 'puppet/util/puppetdb'
+
+class Puppet::Node::Puppetdb < Puppet::Indirector::REST
+  include Puppet::Util::Puppetdb
+
+  def find(request)
+  end
+
+  def save(request)
+  end
+
+  def destroy(request)
+    payload = { :certname => request.key,
+                :producer_timestamp => request.options[:producer_timestamp] || Time.now.iso8601(5) }
+    submit_command(request.key, payload, CommandDeactivateNode, 3)
+  end
+end

data/puppet/lib/puppet/indirector/resource/puppetdb.rb

@@ -0,0 +1,108 @@
+require 'puppet/indirector/rest'
+require 'puppet/util/puppetdb'
+require 'json'
+require 'uri'
+
+class Puppet::Resource::Puppetdb < Puppet::Indirector::REST
+  include Puppet::Util::Puppetdb
+
+  def search(request)
+    profile("resource#search",
+            [:puppetdb, :resource, :search, request.key]) do
+      type   = request.key
+      host   = request.options[:host]
+      filter = request.options[:filter]
+      scope  = request.options[:scope]
+
+      # At minimum, we want to filter to the right type of exported resources.
+      expr = ['and',
+               ['=', 'type', type],
+               ['=', 'exported', true],
+               ['not',
+                 ['=', 'certname', host]]]
+
+      filter_expr = build_expression(filter)
+      expr << filter_expr if filter_expr
+
+      query_param = CGI.escape(expr.to_json)
+
+      begin
+        response = Http.action("/pdb/query/v4/resources?query=#{query_param}") do |http_instance, path|
+          profile("Resources query: #{URI.unescape(path)}",
+                  [:puppetdb, :resource, :search, :query, request.key]) do
+            http_instance.get(path, headers)
+          end
+        end
+
+        log_x_deprecation_header(response)
+
+        unless response.is_a? Net::HTTPSuccess
+          # Newline characters cause an HTTP error, so strip them
+          raise "[#{response.code} #{response.message}] #{response.body.gsub(/[\r\n]/, '')}"
+        end
+      rescue => e
+        raise Puppet::Error, "Could not retrieve resources from the PuppetDB at #{self.class.server}:#{self.class.port}: #{e}"
+      end
+
+      resources = profile("Parse resource query response (size: #{response.body.size})",
+                          [:puppetdb, :resource, :search, :parse_query_response, request.key]) do
+        JSON.load(response.body)
+      end
+
+      profile("Build up collected resource objects (count: #{resources.count})",
+              [:puppetdb, :resource, :search, :build_up_collected_objects, request.key]) do
+        resources.map do |res|
+          params = res['parameters'] || {}
+          params = params.map do |name,value|
+            Puppet::Parser::Resource::Param.new(:name => name, :value => value)
+          end
+          attrs = {:parameters => params, :scope => scope}
+          result = Puppet::Parser::Resource.new(res['type'], res['title'], attrs)
+          result.collector_id = "#{res['certname']}|#{res['type']}|#{res['title']}"
+          result
+        end
+      end
+    end
+  end
+
+  def build_expression(filter)
+    return nil unless filter
+
+    lhs, op, rhs = filter
+
+    case op
+    when '==', '!='
+      build_predicate(op, lhs, rhs)
+    when 'and', 'or'
+      build_join(op, lhs, rhs)
+    else
+      raise Puppet::Error, "Operator #{op} in #{filter.inspect} not supported"
+    end
+  end
+
+  def build_predicate(op, field, value)
+    # Title and tag aren't parameters, so we have to special-case them.
+    expr = case field
+           when "tag"
+             # Tag queries are case-insensitive, so downcase them
+             ["=", "tag", value.downcase]
+           when "title"
+             ["=", "title", value]
+           else
+             ["=", ['parameter', field], value]
+           end
+
+    op == '!=' ? ['not', expr] : expr
+  end
+
+  def build_join(op, lhs, rhs)
+    lhs = build_expression(lhs)
+    rhs = build_expression(rhs)
+
+    [op, lhs, rhs]
+  end
+
+  def headers
+    {'Accept' => 'application/json'}
+  end
+end

data/puppet/lib/puppet/reports/puppetdb.rb

@@ -0,0 +1,188 @@
+require 'puppet'
+require 'puppet/util/puppetdb'
+require 'puppet/util/puppetdb/command_names'
+
+Puppet::Reports.register_report(:puppetdb) do
+  include Puppet::Util::Puppetdb
+
+  CommandStoreReport = Puppet::Util::Puppetdb::CommandNames::CommandStoreReport
+
+  desc <<-DESC
+  Send report information to PuppetDB via the REST API.  Reports are serialized to
+  JSON format, and then submitted to puppetdb using the '#{CommandStoreReport}'
+  command.
+  DESC
+
+  # Process the report by formatting it into a PuppetDB 'store report'
+  # command and submitting it.
+  #
+  # @return [void]
+  def process
+    profile("report#process", [:puppetdb, :report, :process]) do
+      submit_command(self.host, report_to_hash, CommandStoreReport, 6)
+    end
+
+    nil
+  end
+
+  # Convert `self` (an instance of `Puppet::Transaction::Report`) to a hash
+  # suitable for sending over the wire to PuppetDB
+  #
+  # @return Hash[<String, Object>]
+  # @api private
+  def report_to_hash
+    profile("Convert report to wire format hash",
+            [:puppetdb, :report, :convert_to_wire_format_hash]) do
+      if environment.nil?
+        raise Puppet::Error, "Environment is nil, unable to submit report. This may be due a bug with Puppet. Ensure you are running the latest revision, see PUP-2508 for more details."
+      end
+
+      resources = build_resources_list
+      is_noop = resources.any? { |rs| has_noop_event?(rs) } and resources.none? { |rs| has_failed_event?(rs) }
+
+      {
+        "certname"                => host,
+        "puppet_version"          => puppet_version,
+        "report_format"           => report_format,
+        "configuration_version"   => configuration_version.to_s,
+        "producer_timestamp"      => Puppet::Util::Puppetdb.to_wire_time(Time.now),
+        "start_time"              => Puppet::Util::Puppetdb.to_wire_time(time),
+        "end_time"                => Puppet::Util::Puppetdb.to_wire_time(time + run_duration),
+        "environment"             => environment,
+        "transaction_uuid"        => transaction_uuid,
+        "status"                  => status,
+        "noop"                    => is_noop,
+        "logs"                    => build_logs_list,
+        "metrics"                 => build_metrics_list,
+        "resources"               => resources,
+      }
+    end
+  end
+
+  # @return TrueClass
+  # @api private
+  def has_noop_event?(resource)
+    resource["events"].any? { |event| event["status"] == 'noop' }
+  end
+
+  # @return TrueClass
+  # @api private
+  def has_failed_event?(resource)
+    resource["events"].any? { |event| event["status"] == 'failed' }
+  end
+
+  # @return Array[Hash]
+  # @api private
+  def build_resources_list
+    profile("Build resources list (count: #{resource_statuses.count})",
+            [:puppetdb, :resources_list, :build]) do
+      resources = resource_statuses.values.map { |resource| resource_status_to_hash(resource) }
+      if ! config.include_unchanged_resources?
+        resources.select{ |resource| (! resource["events"].empty?) or resource["skipped"] }
+      else
+        resources
+      end
+    end
+  end
+
+  # @return Array[Hash]
+  # @api private
+  def build_logs_list
+    profile("Build logs list (count: #{logs.count})",
+            [:puppetdb, :logs_list, :build]) do
+      logs.map do |log|
+        {
+          'file' => log.file,
+          'line' => log.line,
+          'level' => log.level,
+          'message' => log.message,
+          'source' => log.source,
+          'tags' => [*log.tags],
+          'time' => Puppet::Util::Puppetdb.to_wire_time(log.time),
+        }
+      end
+    end
+  end
+
+  # @return Array[Hash}
+  # @api private
+  def build_metrics_list
+    profile("Build metrics list (count: #{metrics.count})",
+            [:puppetdb, :metrics_list, :build]) do
+      metrics_list = []
+      metrics.each do |name, data|
+        metric_hashes = data.values.map {|x| {"category" => data.name, "name" => x.first, "value" => x.last}}
+        metrics_list.concat(metric_hashes)
+      end
+      metrics_list
+    end
+  end
+
+
+  # @return Number
+  # @api private
+  def run_duration
+    # TODO: this is wrong in puppet.  I am consistently seeing reports where
+    # start-time + this value is less than the timestamp on the individual
+    # resource events.  Not sure what the best short-term fix is yet; the long
+    # term fix is obviously to make the correct data available in puppet.
+    # I've filed a ticket against puppet here:
+    #  http://projects.puppetlabs.com/issues/16480
+    #
+    # NOTE: failed reports have an empty metrics hash. Just send 0 for run time,
+    #  since we don't have access to any better information.
+    if metrics["time"] and metrics["time"]["total"]
+      metrics["time"]["total"]
+    else
+      0
+    end
+  end
+
+  # Convert an instance of `Puppet::Transaction::Event` to a hash
+  # suitable for sending over the wire to PuppetDB
+  #
+  # @return Hash[<String, Object>]
+  # @api private
+  def event_to_hash(event)
+    {
+      "status"            => event.status,
+      "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(event.time),
+      "property"          => event.property,
+      "new_value"         => event.desired_value,
+      "old_value"         => event.previous_value,
+      "message"           => event.message,
+    }
+  end
+
+  def build_events_list(events)
+    profile("Build events list (count: #{events.count})",
+            [:puppetdb, :events_list, :build]) do
+      events.map { |event| event_to_hash(event) }
+    end
+  end
+
+  # Convert an instance of `Puppet::Resource::Status` to a hash
+  # suitable for sending over the wire to PuppetDB
+  #
+  # @return Hash[<String, Object>]
+  # @api private
+  def resource_status_to_hash(resource_status)
+    {
+      "skipped"          => resource_status.skipped,
+      "timestamp"        => Puppet::Util::Puppetdb.to_wire_time(resource_status.time),
+      "resource_type"    => resource_status.resource_type,
+      "resource_title"   => resource_status.title.to_s,
+      "file"             => resource_status.file,
+      "line"             => resource_status.line,
+      "containment_path" => resource_status.containment_path,
+      "events"           => build_events_list(resource_status.events),
+    }
+  end
+
+  # Helper method for accessing the puppetdb configuration
+  #
+  # @api private
+  def config
+    Puppet::Util::Puppetdb.config
+  end
+end

data/puppet/lib/puppet/util/puppetdb.rb

@@ -0,0 +1,108 @@
+require 'puppet/util'
+require 'puppet/util/logging'
+require 'puppet/util/profiler'
+require 'puppet/util/puppetdb/command_names'
+require 'puppet/util/puppetdb/command'
+require 'puppet/util/puppetdb/config'
+require 'digest/sha1'
+require 'time'
+require 'fileutils'
+
+module Puppet::Util::Puppetdb
+
+  class CommandSubmissionError < Puppet::Error
+    def initialize(msg, context)
+      super(msg)
+      @context = context
+    end
+  end
+
+  class InventorySearchError < Puppet::Error
+  end
+  class SoftWriteFailError < Puppet::Error
+  end
+
+
+  def self.config
+    @config ||= Puppet::Util::Puppetdb::Config.load
+    @config
+  end
+
+  def self.puppet3compat?
+    defined?(Puppet::Parser::AST::HashOrArrayAccess)
+  end
+
+  # Given an instance of ruby's Time class, this method converts it to a String
+  # that conforms to PuppetDB's wire format for representing a date/time.
+  def self.to_wire_time(time)
+    # The current implementation simply calls iso8601, but having this method
+    # allows us to change that in the future if needed w/o being forced to
+    # update all of the date objects elsewhere in the code.
+    time.iso8601(9)
+  end
+
+  # Convert a value (usually a string) to a boolean
+  def self.to_bool(value)
+    case value
+    when true, "true"; return true
+    when false, "false"; return false
+    else
+      raise ArgumentError.new("invalid value for Boolean: \"#{val}\"")
+    end
+  end
+
+  # @!group Public instance methods
+
+  # Submit a command to PuppetDB.
+  #
+  # @param certname [String] The certname this command operates on
+  # @param payload [String] payload
+  # @param command_name [String] name of command
+  # @param version [Number] version number of command
+  # @return [Hash <String, String>]
+  def submit_command(certname, payload, command_name, version)
+    profile("Submitted command '#{command_name}' version '#{version}'",
+            [:puppetdb, :command, :submit, command_name, version]) do
+      command = Puppet::Util::Puppetdb::Command.new(command_name, version, certname, payload)
+      command.submit
+    end
+  end
+
+  # Profile a block of code and log the time it took to execute.
+  #
+  # This outputs logs entries to the Puppet masters logging destination
+  # providing the time it took, a message describing the profiled code
+  # and a leaf location marking where the profile method was called
+  # in the profiled hierachy.
+  #
+  # @param message [String] A description of the profiled event
+  # @param metric_id [Array] A list of strings making up the ID of a metric to profile
+  # @param block [Block] The segment of code to profile
+  # @api public
+  def profile(message, metric_id, &block)
+    message = "PuppetDB: " + message
+    arity = Puppet::Util::Profiler.method(:profile).arity
+    case arity
+    when 1
+      Puppet::Util::Profiler.profile(message, &block)
+    when 2, -2
+      Puppet::Util::Profiler.profile(message, metric_id, &block)
+    end
+  end
+
+  # @!group Private instance methods
+
+  # @api private
+  def config
+    Puppet::Util::Puppetdb.config
+  end
+
+  # @api private
+  def log_x_deprecation_header(response)
+    if warning = response['x-deprecation']
+      Puppet.deprecation_warning "Deprecation from PuppetDB: #{warning}"
+    end
+  end
+  module_function :log_x_deprecation_header
+
+end