quixoten-puppetdb-terminus 2.0.0.rc1

data/lib/puppet/indirector/facts/puppetdb_apply.rb

@@ -0,0 +1,25 @@
+require 'puppet/indirector/facts/puppetdb'
+
+# This class provides an alternative implementation of the Facts::Puppetdb
+# terminus that better suits execution via `puppet apply`.
+#
+# This terminus is designed to be used as a cache terminus, to ensure that facts
+# are stored in PuppetDB. It does not act as a real cache itself however, it
+# tells Puppet to fallback to the `terminus` instead.
+class Puppet::Node::Facts::PuppetdbApply < Puppet::Node::Facts::Puppetdb
+  attr_writer :dbstored
+
+  # Here we override the normal save, only saving the first time, as a `save`
+  # can be called multiple times in a puppet run.
+  def save(args)
+    unless @dbstored
+      @dbstored = true
+      super(args)
+    end
+  end
+
+  # By returning nil, we force puppet to use the real terminus.
+  def find(args)
+    nil
+  end
+end

data/lib/puppet/indirector/node/puppetdb.rb

@@ -0,0 +1,22 @@
+require 'puppet/node'
+require 'puppet/indirector/rest'
+require 'puppet/util/puppetdb'
+
+class Puppet::Node::Puppetdb < Puppet::Indirector::REST
+  include Puppet::Util::Puppetdb
+
+  # Run initial checks
+  def initialize
+    Puppet::Util::Puppetdb::GlobalCheck.run
+  end
+
+  def find(request)
+  end
+
+  def save(request)
+  end
+
+  def destroy(request)
+    submit_command(request.key, request.key, CommandDeactivateNode, 2)
+  end
+end

data/lib/puppet/indirector/resource/puppetdb.rb

@@ -0,0 +1,107 @@
+require 'puppet/indirector/rest'
+require 'puppet/util/puppetdb'
+require 'json'
+require 'uri'
+
+class Puppet::Resource::Puppetdb < Puppet::Indirector::REST
+  include Puppet::Util::Puppetdb
+
+  # Run initial checks
+  def initialize
+    Puppet::Util::Puppetdb::GlobalCheck.run
+  end
+
+  def search(request)
+    profile "resource#search" do
+      type   = request.key
+      host   = request.options[:host]
+      filter = request.options[:filter]
+      scope  = request.options[:scope]
+
+      # At minimum, we want to filter to the right type of exported resources.
+      expr = ['and',
+               ['=', 'type', type],
+               ['=', 'exported', true],
+               ['not',
+                 ['=', 'certname', host]]]
+
+      filter_expr = build_expression(filter)
+      expr << filter_expr if filter_expr
+
+      query_param = CGI.escape(expr.to_json)
+
+      begin
+        url = "/v3/resources?query=#{query_param}"
+        response = profile "Resources query: #{URI.unescape(url)}" do
+          http_get(request, url, headers)
+        end
+        log_x_deprecation_header(response)
+
+        unless response.is_a? Net::HTTPSuccess
+          # Newline characters cause an HTTP error, so strip them
+          raise "[#{response.code} #{response.message}] #{response.body.gsub(/[\r\n]/, '')}"
+        end
+      rescue => e
+        raise Puppet::Error, "Could not retrieve resources from the PuppetDB at #{self.class.server}:#{self.class.port}: #{e}"
+      end
+
+      resources = profile "Parse resource query response (size: #{response.body.size})" do
+        JSON.load(response.body)
+      end
+
+      profile "Build up collected resource objects (count: #{resources.count})" do
+        resources.map do |res|
+          params = res['parameters'] || {}
+          params = params.map do |name,value|
+            Puppet::Parser::Resource::Param.new(:name => name, :value => value)
+          end
+          attrs = {:parameters => params, :scope => scope}
+          result = Puppet::Parser::Resource.new(res['type'], res['title'], attrs)
+          result.collector_id = "#{res['certname']}|#{res['type']}|#{res['title']}"
+          result
+        end
+      end
+    end
+  end
+
+  def build_expression(filter)
+    return nil unless filter
+
+    lhs, op, rhs = filter
+
+    case op
+    when '==', '!='
+      build_predicate(op, lhs, rhs)
+    when 'and', 'or'
+      build_join(op, lhs, rhs)
+    else
+      raise Puppet::Error, "Operator #{op} in #{filter.inspect} not supported"
+    end
+  end
+
+  def build_predicate(op, field, value)
+    # Title and tag aren't parameters, so we have to special-case them.
+    expr = case field
+           when "tag"
+             # Tag queries are case-insensitive, so downcase them
+             ["=", "tag", value.downcase]
+           when "title"
+             ["=", "title", value]
+           else
+             ["=", ['parameter', field], value]
+           end
+
+    op == '!=' ? ['not', expr] : expr
+  end
+
+  def build_join(op, lhs, rhs)
+    lhs = build_expression(lhs)
+    rhs = build_expression(rhs)
+
+    [op, lhs, rhs]
+  end
+
+  def headers
+    {'Accept' => 'application/json'}
+  end
+end

data/lib/puppet/reports/puppetdb.rb

@@ -0,0 +1,186 @@
+require 'puppet'
+require 'puppet/util/puppetdb'
+require 'puppet/util/puppetdb/command_names'
+
+Puppet::Reports.register_report(:puppetdb) do
+  include Puppet::Util::Puppetdb
+
+  Puppet::Util::Puppetdb::GlobalCheck.run
+
+  CommandStoreReport = Puppet::Util::Puppetdb::CommandNames::CommandStoreReport
+
+  desc <<-DESC
+  Send report information to PuppetDB via the REST API.  Reports are serialized to
+  JSON format, and then submitted to puppetdb using the '#{CommandStoreReport}'
+  command.
+  DESC
+
+
+  def process
+    profile "report#process" do
+      submit_command(self.host, report_to_hash, CommandStoreReport, 3)
+    end
+  end
+
+  # TODO: It seems unfortunate that we have to access puppet_version and
+  # report_format directly as instance variables.  I've filed the following
+  # ticket / pull req against puppet to expose them via accessors, which
+  # seems more consistent and safer for the long-term.  However, for reasons
+  # relating to backwards compatibility we won't be able to switch over to
+  # the accessors until version 3.x of puppet is our oldest supported version.
+  #
+  # This was resolved in puppet 3.x via ticket #16139 (puppet pull request #1073).
+
+  # @api private
+  def report_format
+    @report_format
+  end
+
+  # @api private
+  def puppet_version
+    @puppet_version
+  end
+
+  # Convert `self` (an instance of `Puppet::Transaction::Report`) to a hash
+  # suitable for sending over the wire to PuppetDB
+  #
+  # @api private
+  def report_to_hash
+    profile "Convert report to wire format hash" do
+      add_v4_fields_to_report(
+        {
+          "certname"                => host,
+          "puppet-version"          => puppet_version,
+          "report-format"           => report_format,
+          "configuration-version"   => configuration_version.to_s,
+          "start-time"              => Puppet::Util::Puppetdb.to_wire_time(time),
+          "end-time"                => Puppet::Util::Puppetdb.to_wire_time(time + run_duration),
+          "resource-events"         => build_events_list,
+          "environment"             => environment,
+        })
+    end
+  end
+
+  # @api private
+  def build_events_list
+    profile "Build events list (count: #{resource_statuses.count})" do
+      filter_events(resource_statuses.inject([]) do |events, status_entry|
+        _, status = *status_entry
+        if ! (status.events.empty?)
+          events.concat(status.events.map { |event| event_to_hash(status, event) })
+        elsif status.skipped
+          events.concat([fabricate_event(status, "skipped")])
+        elsif status.failed
+          # PP-254:
+          #   We have to fabricate resource events here due to a bug/s in report providers
+          #   that causes them not to include events on a resource status that has failed.
+          #   When PuppetDB is able to make a hard break from older version of Puppet that
+          #   have this bug, we can remove this behavior.
+          events.concat([fabricate_event(status, "failure")])
+        end
+        events
+      end)
+    end
+  end
+
+  # @api private
+  def run_duration
+    # TODO: this is wrong in puppet.  I am consistently seeing reports where
+    # start-time + this value is less than the timestamp on the individual
+    # resource events.  Not sure what the best short-term fix is yet; the long
+    # term fix is obviously to make the correct data available in puppet.
+    # I've filed a ticket against puppet here:
+    #  http://projects.puppetlabs.com/issues/16480
+    #
+    # NOTE: failed reports have an empty metrics hash. Just send 0 for run time,
+    #  since we don't have access to any better information.
+    if metrics["time"] and metrics["time"]["total"]
+      metrics["time"]["total"]
+    else
+      raise Puppet::Error, "Report from #{host} contained no metrics, which is often caused by a failed catalog compilation. Unable to process."
+    end
+  end
+
+  # Convert an instance of `Puppet::Transaction::Event` to a hash
+  # suitable for sending over the wire to PuppetDB
+  #
+  # @api private
+  def event_to_hash(resource_status, event)
+    add_v4_fields_to_event(resource_status,
+      {
+        "status"            => event.status,
+        "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(event.time),
+        "resource-type"     => resource_status.resource_type,
+        "resource-title"    => resource_status.title,
+        "property"          => event.property,
+        "new-value"         => event.desired_value,
+        "old-value"         => event.previous_value,
+        "message"           => event.message,
+        "file"              => resource_status.file,
+        "line"              => resource_status.line
+      })
+  end
+
+  # Given an instance of `Puppet::Resource::Status` and a status string,
+  # this method fabricates a PuppetDB event object with the provided
+  # `"status"`.
+  #
+  # @api private
+  def fabricate_event(resource_status, event_status)
+    add_v4_fields_to_event(resource_status,
+      {
+        "status"            => event_status,
+        "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(resource_status.time),
+        "resource-type"     => resource_status.resource_type,
+        "resource-title"    => resource_status.title,
+        "property"          => nil,
+        "new-value"         => nil,
+        "old-value"         => nil,
+        "message"           => nil,
+        "file"              => resource_status.file,
+        "line"              => resource_status.line
+      })
+  end
+
+  # Backwards compatibility with versions of Puppet prior to report format 4
+  #
+  # @api private
+  def add_v4_fields_to_report(report_hash)
+    if report_format >= 4
+      report_hash.merge("transaction-uuid" => transaction_uuid)
+    else
+      report_hash.merge("transaction-uuid" => nil)
+    end
+  end
+
+  # Backwards compatibility with versions of Puppet prior to report format 4
+  #
+  # @api private
+  def add_v4_fields_to_event(resource_status, event_hash)
+    if report_format >= 4
+      event_hash.merge("containment-path" => resource_status.containment_path)
+    else
+      event_hash.merge("containment-path" => nil)
+    end
+  end
+
+  # Filter out blacklisted events, if we're configured to do so
+  #
+  # @api private
+  def filter_events(events)
+    if config.ignore_blacklisted_events?
+      profile "Filter blacklisted events" do
+        events.select { |e| ! config.is_event_blacklisted?(e) }
+      end
+    else
+      events
+    end
+  end
+
+  # Helper method for accessing the puppetdb configuration
+  #
+  # @api private
+  def config
+    Puppet::Util::Puppetdb.config
+  end
+end

data/lib/puppet/util/puppetdb/blacklist.rb

@@ -0,0 +1,35 @@
+module Puppet::Util::Puppetdb
+  class Blacklist
+
+    BlacklistedEvent = Struct.new(:resource_type, :resource_title, :status, :property)
+
+    # Initialize our blacklist of events to filter out of reports.  This is needed
+    # because older versions of puppet always generate a swath of (meaningless)
+    # 'skipped' Schedule events on every agent run.  As of puppet 3.3, these
+    # events should no longer be generated, but this is here for backward compat.
+    BlacklistedEvents =
+        [BlacklistedEvent.new("Schedule", "never", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "puppet", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "hourly", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "daily", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "weekly", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "monthly", "skipped", nil)]
+
+    def initialize(events)
+      @events = events.inject({}) do |m, e|
+        m[e.resource_type] ||= {}
+        m[e.resource_type][e.resource_title] ||= {}
+        m[e.resource_type][e.resource_title][e.status] ||= {}
+        m[e.resource_type][e.resource_title][e.status][e.property] = true
+        m
+      end
+    end
+
+    def is_event_blacklisted?(event)
+      @events.fetch(event["resource-type"], {}).
+        fetch(event["resource-title"], {}).
+        fetch(event["status"], {}).
+        fetch(event["property"], false)
+    end
+  end
+end

data/lib/puppet/util/puppetdb/char_encoding.rb

@@ -0,0 +1,212 @@
+require 'puppet'
+
+module Puppet
+module Util
+module Puppetdb
+module CharEncoding
+
+
+  # Some of this code is modeled after:
+  #  https://github.com/brianmario/utf8/blob/ef10c033/ext/utf8/utf8proc.c
+  #  https://github.com/brianmario/utf8/blob/ef10c033/ext/utf8/string_utf8.c
+
+  Utf8CharLens = [
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+      4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+  ]
+
+  Utf8ReplacementChar = [ 0xEF, 0xBF, 0xBD ].pack("c*")
+
+
+  def self.utf8_string(str)
+    if RUBY_VERSION =~ /1.8/
+      # Ruby 1.8 doesn't have String#encode and related methods, and there
+      #  appears to be a bug in iconv that will interpret some byte sequences
+      #  as 6-byte characters.  Thus, we are forced to resort to some unfortunate
+      #  manual chicanery.
+      warn_if_changed(str, ruby18_clean_utf8(str))
+    elsif str.encoding == Encoding::UTF_8
+      # If we get here, we're in ruby 1.9+, so we have the string encoding methods
+      #  available.  However, just because  a ruby String object is already
+      #  marked as UTF-8, that doesn't guarantee that its contents are actually
+      #  valid; and if you call ruby's ".encode" method with an encoding of
+      #  "utf-8" for a String that ruby already believes is UTF-8, ruby
+      #  seems to optimize that to be a no-op.  So, we have to do some more
+      #  complex handling...
+
+      # If the string already has valid encoding then we're fine.
+      return str if str.valid_encoding?
+
+      # If not, we basically have to walk over the characters and replace
+      #  them by hand.
+      warn_if_changed(str, str.each_char.map { |c| c.valid_encoding? ? c : "\ufffd"}.join)
+    else
+      # if we get here, we're ruby 1.9 and the current string is *not* encoded
+      #  as UTF-8.  Thus we can actually rely on ruby's "encode" method.
+      begin
+        str.encode('UTF-8')
+      rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError => e
+        # If we got an exception, the string is either invalid or not
+        # convertible to UTF-8, so drop those bytes.
+        warn_if_changed(str, str.encode('UTF-8', :invalid => :replace, :undef => :replace))
+      end
+    end
+  end
+
+  # @api private
+  def self.warn_if_changed(str, converted_str)
+    if converted_str != str
+      Puppet.warning "Ignoring invalid UTF-8 byte sequences in data to be sent to PuppetDB"
+    end
+    converted_str
+  end
+
+  # @api private
+  def self.ruby18_clean_utf8(str)
+    #iconv_to_utf8(str)
+    #ruby18_manually_clean_utf8(str)
+
+    # So, we've tried doing this UTF8 cleaning for ruby 1.8 a few different
+    # ways.  Doing it via IConv, we don't do a good job of handling characters
+    # whose codepoints would exceed the legal maximum for UTF-8.  Doing it via
+    # our manual scrubbing process is slower and doesn't catch overlong
+    # encodings.  Since this code really shouldn't even exist in the first place
+    # we've decided to simply compose the two scrubbing methods for now, rather
+    # than trying to add detection of overlong encodings.  It'd be a non-trivial
+    # chunk of code, and it'd have to do a lot of bitwise arithmetic (which Ruby
+    # is not blazingly fast at).
+    ruby18_manually_clean_utf8(iconv_to_utf8(str))
+  end
+
+
+  # @todo we're not using this anymore, but I wanted to leave it around
+  #  for a little while just to make sure that the new code pans out.
+  # @api private
+  def self.iconv_to_utf8(str)
+    iconv = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+
+    # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+    iconv.iconv(str + " ")[0..-2]
+  end
+
+  # @api private
+  def self.get_char_len(byte)
+    Utf8CharLens[byte]
+  end
+
+  # Manually cleans a string by stripping any byte sequences that are
+  # not valid UTF-8 characters.  If you'd prefer for the invalid bytes to be
+  # replaced with the unicode replacement character rather than being stripped,
+  # you may pass `false` for the optional second parameter (`strip`, which
+  # defaults to `true`).
+  #
+  # @api private
+  def self.ruby18_manually_clean_utf8(str, strip = true)
+
+    # This is a hack to allow this code to work with either ruby 1.8 or 1.9,
+    # which is useful for debugging and benchmarking.  For more info see the
+    # comments in the #get_byte method below.
+    @has_get_byte = str.respond_to?(:getbyte)
+
+
+    i = 0
+    len = str.length
+    result = ""
+
+    while i < len
+      byte = get_byte(str, i)
+
+      i += 1
+
+      char_len = get_char_len(byte)
+      case char_len
+      when 0
+        result.concat(Utf8ReplacementChar) unless strip
+      when 1
+        result << byte
+      when 2..4
+        ruby18_handle_multibyte_char(result, byte, str, i,  char_len, strip)
+        i += char_len - 1
+      else
+        raise Puppet::DevError, "Unhandled UTF8 char length: '#{char_len}'"
+      end
+
+    end
+
+    result
+  end
+
+  # @api private
+  def self.ruby18_handle_multibyte_char(result_str, byte, str, i, char_len, strip = true)
+    # keeping an array of bytes for now because we need to do some
+    #  bitwise math on them.
+    char_additional_bytes = []
+
+    # If we don't have enough bytes left to read the full character, we
+    #  put on a replacement character and bail.
+    if i + (char_len - 1) > str.length
+      result_str.concat(Utf8ReplacementChar) unless strip
+      return
+    end
+
+    # we've already read the first byte, so we need to set up a range
+    #  from 0 to (n-2); e.g. if it's a 2-byte char, we will have a range
+    #  from 0 to 0 which will result in reading 1 more byte
+    (0..char_len - 2).each do |x|
+      char_additional_bytes << get_byte(str, i + x)
+    end
+
+    if (is_valid_multibyte_suffix(byte, char_additional_bytes))
+      result_str << byte
+      result_str.concat(char_additional_bytes.pack("c*"))
+    else
+      result_str.concat(Utf8ReplacementChar) unless strip
+    end
+  end
+
+  # @api private
+  def self.is_valid_multibyte_suffix(byte, additional_bytes)
+    # This is heinous, but the UTF-8 spec says that codepoints greater than
+    #  0x10FFFF are illegal.  The first character that is over that limit is
+    #  0xF490bfbf, so if the first byte is F4 then we have to check for
+    #  that condition.
+    if byte == 0xF4
+      val = additional_bytes.inject(0) { |result, b | (result << 8) + b}
+      if val >= 0x90bfbf
+        return false
+      end
+    end
+    additional_bytes.all? { |b| ((b & 0xC0) == 0x80) }
+  end
+
+  # @api private
+  def self.get_byte(str, index)
+    # This method is a hack to allow this code to work with either ruby 1.8
+    #  or 1.9.  In production this code path should never be exercised by
+    #  1.9 because it has a much more sane way to accomplish our goal, but
+    #  for testing, it is useful to be able to run the 1.8 codepath in 1.9.
+    if @has_get_byte
+      str.getbyte(index)
+    else
+      str[index]
+    end
+  end
+
+end
+end
+end
+end