md-puppetdb-terminus 2.0.0.3

data/lib/puppet/util/puppetdb/blacklist.rb

@@ -0,0 +1,35 @@
+module Puppet::Util::Puppetdb
+  class Blacklist
+
+    BlacklistedEvent = Struct.new(:resource_type, :resource_title, :status, :property)
+
+    # Initialize our blacklist of events to filter out of reports.  This is needed
+    # because older versions of puppet always generate a swath of (meaningless)
+    # 'skipped' Schedule events on every agent run.  As of puppet 3.3, these
+    # events should no longer be generated, but this is here for backward compat.
+    BlacklistedEvents =
+        [BlacklistedEvent.new("Schedule", "never", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "puppet", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "hourly", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "daily", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "weekly", "skipped", nil),
+         BlacklistedEvent.new("Schedule", "monthly", "skipped", nil)]
+
+    def initialize(events)
+      @events = events.inject({}) do |m, e|
+        m[e.resource_type] ||= {}
+        m[e.resource_type][e.resource_title] ||= {}
+        m[e.resource_type][e.resource_title][e.status] ||= {}
+        m[e.resource_type][e.resource_title][e.status][e.property] = true
+        m
+      end
+    end
+
+    def is_event_blacklisted?(event)
+      @events.fetch(event["resource-type"], {}).
+        fetch(event["resource-title"], {}).
+        fetch(event["status"], {}).
+        fetch(event["property"], false)
+    end
+  end
+end

data/lib/puppet/util/puppetdb/char_encoding.rb

@@ -0,0 +1,212 @@
+require 'puppet'
+
+module Puppet
+module Util
+module Puppetdb
+module CharEncoding
+
+
+  # Some of this code is modeled after:
+  #  https://github.com/brianmario/utf8/blob/ef10c033/ext/utf8/utf8proc.c
+  #  https://github.com/brianmario/utf8/blob/ef10c033/ext/utf8/string_utf8.c
+
+  Utf8CharLens = [
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+      4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+  ]
+
+  Utf8ReplacementChar = [ 0xEF, 0xBF, 0xBD ].pack("c*")
+
+
+  def self.utf8_string(str)
+    if RUBY_VERSION =~ /1.8/
+      # Ruby 1.8 doesn't have String#encode and related methods, and there
+      #  appears to be a bug in iconv that will interpret some byte sequences
+      #  as 6-byte characters.  Thus, we are forced to resort to some unfortunate
+      #  manual chicanery.
+      warn_if_changed(str, ruby18_clean_utf8(str))
+    elsif str.encoding == Encoding::UTF_8
+      # If we get here, we're in ruby 1.9+, so we have the string encoding methods
+      #  available.  However, just because  a ruby String object is already
+      #  marked as UTF-8, that doesn't guarantee that its contents are actually
+      #  valid; and if you call ruby's ".encode" method with an encoding of
+      #  "utf-8" for a String that ruby already believes is UTF-8, ruby
+      #  seems to optimize that to be a no-op.  So, we have to do some more
+      #  complex handling...
+
+      # If the string already has valid encoding then we're fine.
+      return str if str.valid_encoding?
+
+      # If not, we basically have to walk over the characters and replace
+      #  them by hand.
+      warn_if_changed(str, str.each_char.map { |c| c.valid_encoding? ? c : "\ufffd"}.join)
+    else
+      # if we get here, we're ruby 1.9 and the current string is *not* encoded
+      #  as UTF-8.  Thus we can actually rely on ruby's "encode" method.
+      begin
+        str.encode('UTF-8')
+      rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError => e
+        # If we got an exception, the string is either invalid or not
+        # convertible to UTF-8, so drop those bytes.
+        warn_if_changed(str, str.encode('UTF-8', :invalid => :replace, :undef => :replace))
+      end
+    end
+  end
+
+  # @api private
+  def self.warn_if_changed(str, converted_str)
+    if converted_str != str
+      Puppet.warning "Ignoring invalid UTF-8 byte sequences in data to be sent to PuppetDB"
+    end
+    converted_str
+  end
+
+  # @api private
+  def self.ruby18_clean_utf8(str)
+    #iconv_to_utf8(str)
+    #ruby18_manually_clean_utf8(str)
+
+    # So, we've tried doing this UTF8 cleaning for ruby 1.8 a few different
+    # ways.  Doing it via IConv, we don't do a good job of handling characters
+    # whose codepoints would exceed the legal maximum for UTF-8.  Doing it via
+    # our manual scrubbing process is slower and doesn't catch overlong
+    # encodings.  Since this code really shouldn't even exist in the first place
+    # we've decided to simply compose the two scrubbing methods for now, rather
+    # than trying to add detection of overlong encodings.  It'd be a non-trivial
+    # chunk of code, and it'd have to do a lot of bitwise arithmetic (which Ruby
+    # is not blazingly fast at).
+    ruby18_manually_clean_utf8(iconv_to_utf8(str))
+  end
+
+
+  # @todo we're not using this anymore, but I wanted to leave it around
+  #  for a little while just to make sure that the new code pans out.
+  # @api private
+  def self.iconv_to_utf8(str)
+    iconv = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+
+    # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+    iconv.iconv(str + " ")[0..-2]
+  end
+
+  # @api private
+  def self.get_char_len(byte)
+    Utf8CharLens[byte]
+  end
+
+  # Manually cleans a string by stripping any byte sequences that are
+  # not valid UTF-8 characters.  If you'd prefer for the invalid bytes to be
+  # replaced with the unicode replacement character rather than being stripped,
+  # you may pass `false` for the optional second parameter (`strip`, which
+  # defaults to `true`).
+  #
+  # @api private
+  def self.ruby18_manually_clean_utf8(str, strip = true)
+
+    # This is a hack to allow this code to work with either ruby 1.8 or 1.9,
+    # which is useful for debugging and benchmarking.  For more info see the
+    # comments in the #get_byte method below.
+    @has_get_byte = str.respond_to?(:getbyte)
+
+
+    i = 0
+    len = str.length
+    result = ""
+
+    while i < len
+      byte = get_byte(str, i)
+
+      i += 1
+
+      char_len = get_char_len(byte)
+      case char_len
+      when 0
+        result.concat(Utf8ReplacementChar) unless strip
+      when 1
+        result << byte
+      when 2..4
+        ruby18_handle_multibyte_char(result, byte, str, i,  char_len, strip)
+        i += char_len - 1
+      else
+        raise Puppet::DevError, "Unhandled UTF8 char length: '#{char_len}'"
+      end
+
+    end
+
+    result
+  end
+
+  # @api private
+  def self.ruby18_handle_multibyte_char(result_str, byte, str, i, char_len, strip = true)
+    # keeping an array of bytes for now because we need to do some
+    #  bitwise math on them.
+    char_additional_bytes = []
+
+    # If we don't have enough bytes left to read the full character, we
+    #  put on a replacement character and bail.
+    if i + (char_len - 1) > str.length
+      result_str.concat(Utf8ReplacementChar) unless strip
+      return
+    end
+
+    # we've already read the first byte, so we need to set up a range
+    #  from 0 to (n-2); e.g. if it's a 2-byte char, we will have a range
+    #  from 0 to 0 which will result in reading 1 more byte
+    (0..char_len - 2).each do |x|
+      char_additional_bytes << get_byte(str, i + x)
+    end
+
+    if (is_valid_multibyte_suffix(byte, char_additional_bytes))
+      result_str << byte
+      result_str.concat(char_additional_bytes.pack("c*"))
+    else
+      result_str.concat(Utf8ReplacementChar) unless strip
+    end
+  end
+
+  # @api private
+  def self.is_valid_multibyte_suffix(byte, additional_bytes)
+    # This is heinous, but the UTF-8 spec says that codepoints greater than
+    #  0x10FFFF are illegal.  The first character that is over that limit is
+    #  0xF490bfbf, so if the first byte is F4 then we have to check for
+    #  that condition.
+    if byte == 0xF4
+      val = additional_bytes.inject(0) { |result, b | (result << 8) + b}
+      if val >= 0x90bfbf
+        return false
+      end
+    end
+    additional_bytes.all? { |b| ((b & 0xC0) == 0x80) }
+  end
+
+  # @api private
+  def self.get_byte(str, index)
+    # This method is a hack to allow this code to work with either ruby 1.8
+    #  or 1.9.  In production this code path should never be exercised by
+    #  1.9 because it has a much more sane way to accomplish our goal, but
+    #  for testing, it is useful to be able to run the 1.8 codepath in 1.9.
+    if @has_get_byte
+      str.getbyte(index)
+    else
+      str[index]
+    end
+  end
+
+end
+end
+end
+end

data/lib/puppet/util/puppetdb/command.rb

@@ -0,0 +1,113 @@
+require 'puppet/error'
+require 'puppet/network/http_pool'
+require 'puppet/util/puppetdb'
+require 'puppet/util/puppetdb/command_names'
+require 'puppet/util/puppetdb/char_encoding'
+require 'json'
+
+class Puppet::Util::Puppetdb::Command
+  include Puppet::Util::Puppetdb
+  include Puppet::Util::Puppetdb::CommandNames
+
+  Url                = "/v3/commands"
+
+  # Public instance methods
+
+  # Initialize a Command object, for later submission.
+  #
+  # @param command String the name of the command; should be one of the
+  #   constants defined in `Puppet::Util::Puppetdb::CommandNames`
+  # @param version Integer the command version number
+  # @param certname The certname that this command operates on (is not
+  #   included in the actual submission)
+  # @param payload Object the payload of the command.  This object should be a
+  #   primitive (numeric type, string, array, or hash) that is natively supported
+  #   by JSON serialization / deserialization libraries.
+  def initialize(command, version, certname, payload)
+    @command = command
+    @version = version
+    @certname = certname
+    profile "Format payload" do
+      @payload = self.class.format_payload(command, version, payload)
+    end
+  end
+
+  attr_reader :command, :version, :certname, :payload
+
+  # Submit the command, returning the result hash.
+  #
+  # @return [Hash <String, String>]
+  def submit
+    checksum = Digest::SHA1.hexdigest(payload)
+
+    for_whom = " for #{certname}" if certname
+
+    begin
+      response = profile "Submit command HTTP post" do
+        http = Puppet::Network::HttpPool.http_instance(config.server, config.port)
+        http.post(Url + "?checksum=#{checksum}", payload, headers)
+      end
+
+      Puppet::Util::Puppetdb.log_x_deprecation_header(response)
+
+      if response.is_a? Net::HTTPSuccess
+        result = JSON.parse(response.body)
+        Puppet.info "'#{command}' command#{for_whom} submitted to PuppetDB with UUID #{result['uuid']}"
+        result
+      else
+        # Newline characters cause an HTTP error, so strip them
+        error = "[#{response.code} #{response.message}] #{response.body.gsub(/[\r\n]/, '')}"
+        if config.soft_write_failure
+          Puppet.err "'#{command}'command#{for_whom} failed during submission to PuppetDB: #{error}"
+        else
+          raise Puppet::Error, error
+        end
+      end
+    rescue => e
+      error = "Failed to submit '#{command}' command#{for_whom} to PuppetDB at #{config.server}:#{config.port}: #{e}"
+      if config.soft_write_failure
+        Puppet.err error
+      else
+        # TODO: Use new exception handling methods from Puppet 3.0 here as soon as
+        #  we are able to do so (can't call them yet w/o breaking backwards
+        #  compatibility.)  We should either be using a nested exception or calling
+        #  Puppet::Util::Logging#log_exception or #log_and_raise here; w/o them
+        #  we lose context as to where the original exception occurred.
+        puts e, e.backtrace if Puppet[:trace]
+        raise Puppet::Error, error
+      end
+    end
+  end
+
+
+  # @!group Private class methods
+
+  # @api private
+  def self.format_payload(command, version, payload)
+    message = {
+      :command => command,
+      :version => version,
+      :payload => payload,
+    }.to_pson
+
+    Puppet::Util::Puppetdb::CharEncoding.utf8_string(message)
+  end
+
+  # @!group Private instance methods
+
+  # @api private
+  def headers
+    {
+      "Accept" => "application/json",
+      "Content-Type" => "application/json",
+    }
+  end
+
+  # @api private
+  def config
+    # Would prefer to pass this to the constructor or acquire it some other
+    # way besides this pseudo-global reference.
+    Puppet::Util::Puppetdb.config
+  end
+
+end

data/lib/puppet/util/puppetdb/command_names.rb

@@ -0,0 +1,8 @@
+module Puppet::Util::Puppetdb
+  module CommandNames
+    CommandReplaceCatalog   = "replace catalog"
+    CommandReplaceFacts     = "replace facts"
+    CommandDeactivateNode   = "deactivate node"
+    CommandStoreReport      = "store report"
+  end
+end

data/lib/puppet/util/puppetdb/config.rb

@@ -0,0 +1,112 @@
+require 'puppet/util/puppetdb/command_names'
+require 'puppet/util/puppetdb/blacklist'
+
+module Puppet::Util::Puppetdb
+class Config
+  include Puppet::Util::Puppetdb::CommandNames
+
+  # Public class methods
+
+  def self.load(config_file = nil)
+    defaults = {
+      :server                    => "puppetdb",
+      :port                      => 8081,
+      :soft_write_failure        => false,
+      :ignore_blacklisted_events => true,
+    }
+
+    config_file ||= File.join(Puppet[:confdir], "puppetdb.conf")
+
+    if File.exists?(config_file)
+      Puppet.debug("Configuring PuppetDB terminuses with config file #{config_file}")
+      content = File.read(config_file)
+    else
+      Puppet.debug("No #{config_file} file found; falling back to default server and port #{defaults[:server]}:#{defaults[:port]}")
+      content = ''
+    end
+
+    result = {}
+    section = nil
+    content.lines.each_with_index do |line,number|
+      # Gotta track the line numbers properly
+      number += 1
+      case line
+        when /^\[(\w+)\s*\]$/
+          section = $1
+          result[section] ||= {}
+        when /^\s*(\w+)\s*=\s*(\S+)\s*$/
+          raise "Setting '#{line}' is illegal outside of section in PuppetDB config #{config_file}:#{number}" unless section
+          result[section][$1] = $2
+        when /^\s*[#;]/
+          # Skip comments
+        when /^\s*$/
+          # Skip blank lines
+        else
+          raise "Unparseable line '#{line}' in PuppetDB config #{config_file}:#{number}"
+      end
+    end
+
+
+    main_section = result['main'] || {}
+    # symbolize the keys
+    main_section = main_section.inject({}) {|h, (k,v)| h[k.to_sym] = v ; h}
+    # merge with defaults but filter out anything except the legal settings
+    config_hash = defaults.merge(main_section).reject do |k, v|
+      !([:server, :port, :ignore_blacklisted_events, :soft_write_failure].include?(k))
+    end
+
+    config_hash[:server] = config_hash[:server].strip
+    config_hash[:port] = config_hash[:port].to_i
+    config_hash[:ignore_blacklisted_events] =
+      Puppet::Util::Puppetdb.to_bool(config_hash[:ignore_blacklisted_events])
+    config_hash[:soft_write_failure] =
+      Puppet::Util::Puppetdb.to_bool(config_hash[:soft_write_failure])
+
+    self.new(config_hash)
+  rescue => detail
+    puts detail.backtrace if Puppet[:trace]
+    Puppet.warning "Could not configure PuppetDB terminuses: #{detail}"
+    raise
+  end
+
+  # @!group Public instance methods
+
+  def initialize(config_hash = {})
+    @config = config_hash
+    initialize_blacklisted_events()
+  end
+
+  def server
+    config[:server]
+  end
+
+  def port
+    config[:port]
+  end
+
+  def ignore_blacklisted_events?
+    config[:ignore_blacklisted_events]
+  end
+
+  def is_event_blacklisted?(event)
+   @blacklist.is_event_blacklisted? event
+  end
+
+  def soft_write_failure
+    config[:soft_write_failure]
+  end
+
+  # @!group Private instance methods
+
+  # @!attribute [r] count
+  #   @api private
+  attr_reader :config
+
+  Blacklist = Puppet::Util::Puppetdb::Blacklist
+
+  # @api private
+  def initialize_blacklisted_events(events = Blacklist::BlacklistedEvents)
+    @blacklist = Blacklist.new(events)
+  end
+end
+end