md-puppetdb-terminus 2.0.0.3

data/lib/puppet/indirector/facts/puppetdb_apply.rb

@@ -0,0 +1,25 @@
+require 'puppet/indirector/facts/puppetdb'
+
+# This class provides an alternative implementation of the Facts::Puppetdb
+# terminus that better suits execution via `puppet apply`.
+#
+# This terminus is designed to be used as a cache terminus, to ensure that facts
+# are stored in PuppetDB. It does not act as a real cache itself however, it
+# tells Puppet to fallback to the `terminus` instead.
+class Puppet::Node::Facts::PuppetdbApply < Puppet::Node::Facts::Puppetdb
+  attr_writer :dbstored
+
+  # Here we override the normal save, only saving the first time, as a `save`
+  # can be called multiple times in a puppet run.
+  def save(args)
+    unless @dbstored
+      @dbstored = true
+      super(args)
+    end
+  end
+
+  # By returning nil, we force puppet to use the real terminus.
+  def find(args)
+    nil
+  end
+end

data/lib/puppet/indirector/node/puppetdb.rb

@@ -0,0 +1,22 @@
+require 'puppet/node'
+require 'puppet/indirector/rest'
+require 'puppet/util/puppetdb'
+
+class Puppet::Node::Puppetdb < Puppet::Indirector::REST
+  include Puppet::Util::Puppetdb
+
+  # Run initial checks
+  def initialize
+    Puppet::Util::Puppetdb::GlobalCheck.run
+  end
+
+  def find(request)
+  end
+
+  def save(request)
+  end
+
+  def destroy(request)
+    submit_command(request.key, request.key, CommandDeactivateNode, 2)
+  end
+end

data/lib/puppet/indirector/resource/puppetdb.rb

@@ -0,0 +1,107 @@
+require 'puppet/indirector/rest'
+require 'puppet/util/puppetdb'
+require 'json'
+require 'uri'
+
+class Puppet::Resource::Puppetdb < Puppet::Indirector::REST
+  include Puppet::Util::Puppetdb
+
+  # Run initial checks
+  def initialize
+    Puppet::Util::Puppetdb::GlobalCheck.run
+  end
+
+  def search(request)
+    profile "resource#search" do
+      type   = request.key
+      host   = request.options[:host]
+      filter = request.options[:filter]
+      scope  = request.options[:scope]
+
+      # At minimum, we want to filter to the right type of exported resources.
+      expr = ['and',
+               ['=', 'type', type],
+               ['=', 'exported', true],
+               ['not',
+                 ['=', 'certname', host]]]
+
+      filter_expr = build_expression(filter)
+      expr << filter_expr if filter_expr
+
+      query_param = CGI.escape(expr.to_json)
+
+      begin
+        url = "/v3/resources?query=#{query_param}"
+        response = profile "Resources query: #{URI.unescape(url)}" do
+          http_get(request, url, headers)
+        end
+        log_x_deprecation_header(response)
+
+        unless response.is_a? Net::HTTPSuccess
+          # Newline characters cause an HTTP error, so strip them
+          raise "[#{response.code} #{response.message}] #{response.body.gsub(/[\r\n]/, '')}"
+        end
+      rescue => e
+        raise Puppet::Error, "Could not retrieve resources from the PuppetDB at #{self.class.server}:#{self.class.port}: #{e}"
+      end
+
+      resources = profile "Parse resource query response (size: #{response.body.size})" do
+        JSON.load(response.body)
+      end
+
+      profile "Build up collected resource objects (count: #{resources.count})" do
+        resources.map do |res|
+          params = res['parameters'] || {}
+          params = params.map do |name,value|
+            Puppet::Parser::Resource::Param.new(:name => name, :value => value)
+          end
+          attrs = {:parameters => params, :scope => scope}
+          result = Puppet::Parser::Resource.new(res['type'], res['title'], attrs)
+          result.collector_id = "#{res['certname']}|#{res['type']}|#{res['title']}"
+          result
+        end
+      end
+    end
+  end
+
+  def build_expression(filter)
+    return nil unless filter
+
+    lhs, op, rhs = filter
+
+    case op
+    when '==', '!='
+      build_predicate(op, lhs, rhs)
+    when 'and', 'or'
+      build_join(op, lhs, rhs)
+    else
+      raise Puppet::Error, "Operator #{op} in #{filter.inspect} not supported"
+    end
+  end
+
+  def build_predicate(op, field, value)
+    # Title and tag aren't parameters, so we have to special-case them.
+    expr = case field
+           when "tag"
+             # Tag queries are case-insensitive, so downcase them
+             ["=", "tag", value.downcase]
+           when "title"
+             ["=", "title", value]
+           else
+             ["=", ['parameter', field], value]
+           end
+
+    op == '!=' ? ['not', expr] : expr
+  end
+
+  def build_join(op, lhs, rhs)
+    lhs = build_expression(lhs)
+    rhs = build_expression(rhs)
+
+    [op, lhs, rhs]
+  end
+
+  def headers
+    {'Accept' => 'application/json'}
+  end
+end

data/lib/puppet/reports/puppetdb.rb

@@ -0,0 +1,186 @@
+require 'puppet'
+require 'puppet/util/puppetdb'
+require 'puppet/util/puppetdb/command_names'
+
+Puppet::Reports.register_report(:puppetdb) do
+  include Puppet::Util::Puppetdb
+
+  Puppet::Util::Puppetdb::GlobalCheck.run
+
+  CommandStoreReport = Puppet::Util::Puppetdb::CommandNames::CommandStoreReport
+
+  desc <<-DESC
+  Send report information to PuppetDB via the REST API.  Reports are serialized to
+  JSON format, and then submitted to puppetdb using the '#{CommandStoreReport}'
+  command.
+  DESC
+
+
+  def process
+    profile "report#process" do
+      submit_command(self.host, report_to_hash, CommandStoreReport, 3)
+    end
+  end
+
+  # TODO: It seems unfortunate that we have to access puppet_version and
+  # report_format directly as instance variables.  I've filed the following
+  # ticket / pull req against puppet to expose them via accessors, which
+  # seems more consistent and safer for the long-term.  However, for reasons
+  # relating to backwards compatibility we won't be able to switch over to
+  # the accessors until version 3.x of puppet is our oldest supported version.
+  #
+  # This was resolved in puppet 3.x via ticket #16139 (puppet pull request #1073).
+
+  # @api private
+  def report_format
+    @report_format
+  end
+
+  # @api private
+  def puppet_version
+    @puppet_version
+  end
+
+  # Convert `self` (an instance of `Puppet::Transaction::Report`) to a hash
+  # suitable for sending over the wire to PuppetDB
+  #
+  # @api private
+  def report_to_hash
+    profile "Convert report to wire format hash" do
+      add_v4_fields_to_report(
+        {
+          "certname"                => host,
+          "puppet-version"          => puppet_version,
+          "report-format"           => report_format,
+          "configuration-version"   => configuration_version.to_s,
+          "start-time"              => Puppet::Util::Puppetdb.to_wire_time(time),
+          "end-time"                => Puppet::Util::Puppetdb.to_wire_time(time + run_duration),
+          "resource-events"         => build_events_list,
+          "environment"             => environment,
+        })
+    end
+  end
+
+  # @api private
+  def build_events_list
+    profile "Build events list (count: #{resource_statuses.count})" do
+      filter_events(resource_statuses.inject([]) do |events, status_entry|
+        _, status = *status_entry
+        if ! (status.events.empty?)
+          events.concat(status.events.map { |event| event_to_hash(status, event) })
+        elsif status.skipped
+          events.concat([fabricate_event(status, "skipped")])
+        elsif status.failed
+          # PP-254:
+          #   We have to fabricate resource events here due to a bug/s in report providers
+          #   that causes them not to include events on a resource status that has failed.
+          #   When PuppetDB is able to make a hard break from older version of Puppet that
+          #   have this bug, we can remove this behavior.
+          events.concat([fabricate_event(status, "failure")])
+        end
+        events
+      end)
+    end
+  end
+
+  # @api private
+  def run_duration
+    # TODO: this is wrong in puppet.  I am consistently seeing reports where
+    # start-time + this value is less than the timestamp on the individual
+    # resource events.  Not sure what the best short-term fix is yet; the long
+    # term fix is obviously to make the correct data available in puppet.
+    # I've filed a ticket against puppet here:
+    #  http://projects.puppetlabs.com/issues/16480
+    #
+    # NOTE: failed reports have an empty metrics hash. Just send 0 for run time,
+    #  since we don't have access to any better information.
+    if metrics["time"] and metrics["time"]["total"]
+      metrics["time"]["total"]
+    else
+      raise Puppet::Error, "Report from #{host} contained no metrics, which is often caused by a failed catalog compilation. Unable to process."
+    end
+  end
+
+  # Convert an instance of `Puppet::Transaction::Event` to a hash
+  # suitable for sending over the wire to PuppetDB
+  #
+  # @api private
+  def event_to_hash(resource_status, event)
+    add_v4_fields_to_event(resource_status,
+      {
+        "status"            => event.status,
+        "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(event.time),
+        "resource-type"     => resource_status.resource_type,
+        "resource-title"    => resource_status.title,
+        "property"          => event.property,
+        "new-value"         => event.desired_value,
+        "old-value"         => event.previous_value,
+        "message"           => event.message,
+        "file"              => resource_status.file,
+        "line"              => resource_status.line
+      })
+  end
+
+  # Given an instance of `Puppet::Resource::Status` and a status string,
+  # this method fabricates a PuppetDB event object with the provided
+  # `"status"`.
+  #
+  # @api private
+  def fabricate_event(resource_status, event_status)
+    add_v4_fields_to_event(resource_status,
+      {
+        "status"            => event_status,
+        "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(resource_status.time),
+        "resource-type"     => resource_status.resource_type,
+        "resource-title"    => resource_status.title,
+        "property"          => nil,
+        "new-value"         => nil,
+        "old-value"         => nil,
+        "message"           => nil,
+        "file"              => resource_status.file,
+        "line"              => resource_status.line
+      })
+  end
+
+  # Backwards compatibility with versions of Puppet prior to report format 4
+  #
+  # @api private
+  def add_v4_fields_to_report(report_hash)
+    if report_format >= 4
+      report_hash.merge("transaction-uuid" => transaction_uuid)
+    else
+      report_hash.merge("transaction-uuid" => nil)
+    end
+  end
+
+  # Backwards compatibility with versions of Puppet prior to report format 4
+  #
+  # @api private
+  def add_v4_fields_to_event(resource_status, event_hash)
+    if report_format >= 4
+      event_hash.merge("containment-path" => resource_status.containment_path)
+    else
+      event_hash.merge("containment-path" => nil)
+    end
+  end
+
+  # Filter out blacklisted events, if we're configured to do so
+  #
+  # @api private
+  def filter_events(events)
+    if config.ignore_blacklisted_events?
+      profile "Filter blacklisted events" do
+        events.select { |e| ! config.is_event_blacklisted?(e) }
+      end
+    else
+      events
+    end
+  end
+
+  # Helper method for accessing the puppetdb configuration
+  #
+  # @api private
+  def config
+    Puppet::Util::Puppetdb.config
+  end
+end

data/lib/puppet/util/puppetdb.rb

@@ -0,0 +1,108 @@
+require 'puppet/util'
+require 'puppet/util/logging'
+require 'puppet/util/profiler'
+require 'puppet/util/puppetdb/global_check'
+require 'puppet/util/puppetdb/command_names'
+require 'puppet/util/puppetdb/command'
+require 'puppet/util/puppetdb/config'
+require 'digest/sha1'
+require 'time'
+require 'fileutils'
+
+module Puppet::Util::Puppetdb
+
+  def self.server
+    config.server
+  end
+
+  def self.port
+    config.port
+  end
+
+  def self.config
+    @config ||= Puppet::Util::Puppetdb::Config.load
+    @config
+  end
+
+  # This magical stuff is needed so that the indirector termini will make requests to
+  # the correct host/port, because this module gets mixed in to our indirector
+  # termini.
+  module ClassMethods
+    def server
+      Puppet::Util::Puppetdb.server
+    end
+
+    def port
+      Puppet::Util::Puppetdb.port
+    end
+  end
+
+  def self.included(child)
+    child.extend ClassMethods
+  end
+
+  # Given an instance of ruby's Time class, this method converts it to a String
+  # that conforms to PuppetDB's wire format for representing a date/time.
+  def self.to_wire_time(time)
+    # The current implementation simply calls iso8601, but having this method
+    # allows us to change that in the future if needed w/o being forced to
+    # update all of the date objects elsewhere in the code.
+    time.iso8601(9)
+  end
+
+  # Convert a value (usually a string) to a boolean
+  def self.to_bool(value)
+    case value
+    when true, "true"; return true
+    when false, "false"; return false
+    else
+      raise ArgumentError.new("invalid value for Boolean: \"#{val}\"")
+    end
+  end
+
+  # @!group Public instance methods
+
+  # Submit a command to PuppetDB.
+  #
+  # @param certname [String] hostname name of puppetdb instance
+  # @param payload [String] payload
+  # @param command_name [String] name of command
+  # @param version [Number] version number of command
+  def submit_command(certname, payload, command_name, version)
+    profile "Submitted command '#{command_name}' version '#{version}'" do
+      command = Puppet::Util::Puppetdb::Command.new(command_name, version, certname, payload)
+      command.submit
+    end
+  end
+
+  # Profile a block of code and log the time it took to execute.
+  #
+  # This outputs logs entries to the Puppet masters logging destination
+  # providing the time it took, a message describing the profiled code
+  # and a leaf location marking where the profile method was called
+  # in the profiled hierachy.
+  #
+  # @param message [String] A description of the profiled event
+  # @param block [Block] The segment of code to profile
+  # @api public
+  def profile(message, &block)
+    message = "PuppetDB: " + message
+    Puppet::Util::Profiler.profile(message, &block)
+  end
+
+  # @!group Private instance methods
+
+  # @api private
+  def config
+    Puppet::Util::Puppetdb.config
+  end
+
+  # @api private
+  def log_x_deprecation_header(response)
+    if warning = response['x-deprecation']
+      Puppet.deprecation_warning "Deprecation from PuppetDB: #{warning}"
+    end
+  end
+  module_function :log_x_deprecation_header
+
+end