ml-puppetdb-terminus 3.2.1

data/puppet/lib/puppet/util/puppetdb/char_encoding.rb

@@ -0,0 +1,316 @@
+require 'puppet'
+
+module Puppet
+module Util
+module Puppetdb
+module CharEncoding
+
+
+  # Some of this code is modeled after:
+  #  https://github.com/brianmario/utf8/blob/ef10c033/ext/utf8/utf8proc.c
+  #  https://github.com/brianmario/utf8/blob/ef10c033/ext/utf8/string_utf8.c
+
+  Utf8CharLens = [
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+      4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+  ]
+
+  Utf8ReplacementChar = [ 0xEF, 0xBF, 0xBD ].pack("c*")
+
+  DEFAULT_INVALID_CHAR = "\ufffd"
+
+  # @api private
+  def self.all_indexes_of_char(str, char)
+    (0..str.length).find_all{ |i| str[i] == char}
+  end
+
+  # @api private
+  #
+  # Takes an array and returns a sub-array without the last element
+  #
+  # @return [Object]
+  def self.drop_last(array)
+    array[0..-2]
+  end
+
+  # @api private
+  #
+  # Takes an array of increasing integers and collapses the sequential
+  # integers into ranges
+  #
+  # @param index_array an array of sorted integers
+  # @return [Range]
+  def self.collapse_ranges(index_array)
+    ranges = index_array.each.inject([]) do |spans, n|
+      if spans.empty? || spans.last.end != n - 1
+        spans << Range.new(n, n)
+      else
+        drop_last(spans) << Range.new(spans.last.begin,n)
+      end
+    end
+  end
+
+  # @api private
+  #
+  # Scans the string s with bad characters found at bad_char_indexes
+  # and returns an array of messages that give some context around the
+  # bad characters. This will give up to 100 characters prior to the
+  # bad character and 100 after. It will return fewer if it's at the
+  # beginning of a string or if another bad character appears before
+  # reaching the 100 characters
+  #
+  # @param str string coming from to_pson, likely a command to be submitted to PDB
+  # @param bad_char_indexes an array of indexes into the string where invalid characters were found
+  # @return [String]
+  def self.error_char_context(str, bad_char_indexes)
+    bad_char_ranges = collapse_ranges(bad_char_indexes)
+    bad_char_ranges.each_with_index.inject([]) do |state, (r, index)|
+      gap = r.to_a.length
+
+      prev_bad_char_end = bad_char_ranges[index-1].end + 1 if index > 0
+      next_bad_char_begin = bad_char_ranges[index+1].begin - 1 if index < bad_char_ranges.length - 1
+
+      start_char = [prev_bad_char_end || 0, r.begin-100].max
+      end_char = [next_bad_char_begin || str.length - 1, r.end+100].min
+      x = [next_bad_char_begin || str.length, r.end+100, str.length]
+      prefix = str[start_char..r.begin-1]
+      suffix = str[r.end+1..end_char]
+
+      state << "'#{prefix}' followed by #{gap} invalid/undefined bytes then '#{suffix}'"
+    end
+  end
+
+  # @api private
+  #
+  # Warns the user if an invalid character was found. If debugging is
+  # enabled will also log contextual information about where the bad
+  # character(s) were found
+  #
+  # @param str A string coming from to_pson, likely a command to be submitted to PDB
+  # @param error_context_str information about where this string came from for use in error messages
+  # @return String
+  def self.warn_if_invalid_chars(str, error_context_str)
+    bad_char_indexes = all_indexes_of_char(str, DEFAULT_INVALID_CHAR)
+    if bad_char_indexes.empty?
+      str
+    else
+      Puppet.warning "#{error_context_str} ignoring invalid UTF-8 byte sequences in data to be sent to PuppetDB, see debug logging for more info"
+      if Puppet.settings[:log_level] == "debug"
+        Puppet.debug error_context_str + "\n" + error_char_context(str, bad_char_indexes).join("\n")
+      end
+
+      str
+    end
+  end
+
+  # @api private
+  #
+  # Attempts to coerce str to UTF-8, if that fails will output context
+  # information using error_context_str
+  #
+  # @param str A string coming from to_pson, likely a command to be submitted to PDB
+  # @param error_context_str information about where this string came from for use in error messages
+  # @return Str
+  def self.coerce_to_utf8(str, error_context_str)
+    str_copy = str.dup
+    # This code is passed in a string that was created by
+    # to_pson. to_pson calls force_encoding('ASCII-8BIT') on the
+    # string before it returns it. This leaves the actual UTF-8 bytes
+    # alone. Below we check to see if this is the case (this should be
+    # most common). In this case, the bytes are still UTF-8 and we can
+    # just encode! and we're good to go. If They are not valid UTF-8
+    # bytes, that means there is probably some binary data mixed in
+    # the middle of the UTF-8 string. In this case we need to output a
+    # warning and give the user more information
+    str_copy.force_encoding("UTF-8")
+    if str_copy.valid_encoding?
+      str_copy.encode!("UTF-8")
+    else
+      # This is force_encoded as US-ASCII to avoid any overlapping
+      # byte related issues that could arise from mis-interpreting a
+      # random extra byte as part of a multi-byte UTF-8 character
+      str_copy.force_encoding("US-ASCII")
+      warn_if_invalid_chars(str_copy.encode!("UTF-8",
+                                             :invalid => :replace,
+                                             :undef => :replace,
+                                             :replace => DEFAULT_INVALID_CHAR),
+                            error_context_str)
+    end
+  end
+
+  def self.utf8_string(str, error_context_str)
+    if RUBY_VERSION =~ /^1.8/
+      # Ruby 1.8 doesn't have String#encode and related methods, and there
+      #  appears to be a bug in iconv that will interpret some byte sequences
+      #  as 6-byte characters.  Thus, we are forced to resort to some unfortunate
+      #  manual chicanery.
+      warn_if_changed(str, ruby18_clean_utf8(str))
+    else
+      begin
+        coerce_to_utf8(str, error_context_str)
+      rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError => e
+          # If we got an exception, the string is either invalid or not
+          # convertible to UTF-8, so drop those bytes.
+
+        warn_if_changed(str, str.encode('UTF-8', :invalid => :replace, :undef => :replace))
+      end
+    end
+  end
+
+  # @api private
+  def self.warn_if_changed(str, converted_str)
+    if converted_str != str
+      Puppet.warning "Ignoring invalid UTF-8 byte sequences in data to be sent to PuppetDB"
+    end
+    converted_str
+  end
+
+  # @api private
+  def self.ruby18_clean_utf8(str)
+    #iconv_to_utf8(str)
+    #ruby18_manually_clean_utf8(str)
+
+    # So, we've tried doing this UTF8 cleaning for ruby 1.8 a few different
+    # ways.  Doing it via IConv, we don't do a good job of handling characters
+    # whose codepoints would exceed the legal maximum for UTF-8.  Doing it via
+    # our manual scrubbing process is slower and doesn't catch overlong
+    # encodings.  Since this code really shouldn't even exist in the first place
+    # we've decided to simply compose the two scrubbing methods for now, rather
+    # than trying to add detection of overlong encodings.  It'd be a non-trivial
+    # chunk of code, and it'd have to do a lot of bitwise arithmetic (which Ruby
+    # is not blazingly fast at).
+    ruby18_manually_clean_utf8(iconv_to_utf8(str))
+  end
+
+
+  # @todo we're not using this anymore, but I wanted to leave it around
+  #  for a little while just to make sure that the new code pans out.
+  # @api private
+  def self.iconv_to_utf8(str)
+    iconv = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+
+    # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+    iconv.iconv(str + " ")[0..-2]
+  end
+
+  # @api private
+  def self.get_char_len(byte)
+    Utf8CharLens[byte]
+  end
+
+  # Manually cleans a string by stripping any byte sequences that are
+  # not valid UTF-8 characters.  If you'd prefer for the invalid bytes to be
+  # replaced with the unicode replacement character rather than being stripped,
+  # you may pass `false` for the optional second parameter (`strip`, which
+  # defaults to `true`).
+  #
+  # @api private
+  def self.ruby18_manually_clean_utf8(str, strip = true)
+
+    # This is a hack to allow this code to work with either ruby 1.8 or 1.9,
+    # which is useful for debugging and benchmarking.  For more info see the
+    # comments in the #get_byte method below.
+    @has_get_byte = str.respond_to?(:getbyte)
+
+
+    i = 0
+    len = str.length
+    result = ""
+
+    while i < len
+      byte = get_byte(str, i)
+
+      i += 1
+
+      char_len = get_char_len(byte)
+      case char_len
+      when 0
+        result.concat(Utf8ReplacementChar) unless strip
+      when 1
+        result << byte
+      when 2..4
+        ruby18_handle_multibyte_char(result, byte, str, i,  char_len, strip)
+        i += char_len - 1
+      else
+        raise Puppet::DevError, "Unhandled UTF8 char length: '#{char_len}'"
+      end
+
+    end
+
+    result
+  end
+
+  # @api private
+  def self.ruby18_handle_multibyte_char(result_str, byte, str, i, char_len, strip = true)
+    # keeping an array of bytes for now because we need to do some
+    #  bitwise math on them.
+    char_additional_bytes = []
+
+    # If we don't have enough bytes left to read the full character, we
+    #  put on a replacement character and bail.
+    if i + (char_len - 1) > str.length
+      result_str.concat(Utf8ReplacementChar) unless strip
+      return
+    end
+
+    # we've already read the first byte, so we need to set up a range
+    #  from 0 to (n-2); e.g. if it's a 2-byte char, we will have a range
+    #  from 0 to 0 which will result in reading 1 more byte
+    (0..char_len - 2).each do |x|
+      char_additional_bytes << get_byte(str, i + x)
+    end
+
+    if (is_valid_multibyte_suffix(byte, char_additional_bytes))
+      result_str << byte
+      result_str.concat(char_additional_bytes.pack("c*"))
+    else
+      result_str.concat(Utf8ReplacementChar) unless strip
+    end
+  end
+
+  # @api private
+  def self.is_valid_multibyte_suffix(byte, additional_bytes)
+    # This is heinous, but the UTF-8 spec says that codepoints greater than
+    #  0x10FFFF are illegal.  The first character that is over that limit is
+    #  0xF490bfbf, so if the first byte is F4 then we have to check for
+    #  that condition.
+    if byte == 0xF4
+      val = additional_bytes.inject(0) { |result, b | (result << 8) + b}
+      if val >= 0x90bfbf
+        return false
+      end
+    end
+    additional_bytes.all? { |b| ((b & 0xC0) == 0x80) }
+  end
+
+  # @api private
+  def self.get_byte(str, index)
+    # This method is a hack to allow this code to work with either ruby 1.8
+    #  or 1.9.  In production this code path should never be exercised by
+    #  1.9 because it has a much more sane way to accomplish our goal, but
+    #  for testing, it is useful to be able to run the 1.8 codepath in 1.9.
+    if @has_get_byte
+      str.getbyte(index)
+    else
+      str[index]
+    end
+  end
+
+end
+end
+end
+end

data/puppet/lib/puppet/util/puppetdb/command.rb

@@ -0,0 +1,116 @@
+require 'puppet/error'
+require 'puppet/util/puppetdb'
+require 'puppet/util/puppetdb/http'
+require 'puppet/util/puppetdb/command_names'
+require 'puppet/util/puppetdb/char_encoding'
+require 'json'
+
+class Puppet::Util::Puppetdb::Command
+  include Puppet::Util::Puppetdb
+  include Puppet::Util::Puppetdb::CommandNames
+
+  CommandsUrl = "/pdb/cmd/v1"
+
+  # Public instance methods
+
+  # Initialize a Command object, for later submission.
+  #
+  # @param command String the name of the command; should be one of the
+  #   constants defined in `Puppet::Util::Puppetdb::CommandNames`
+  # @param version Integer the command version number
+  # @param certname The certname that this command operates on (is not
+  #   included in the actual submission)
+  # @param payload Object the payload of the command.  This object should be a
+  #   primitive (numeric type, string, array, or hash) that is natively supported
+  #   by JSON serialization / deserialization libraries.
+  def initialize(command, version, certname, payload)
+    @command = command
+    @version = version
+    @certname = certname
+    profile("Format payload", [:puppetdb, :payload, :format]) do
+      @payload = Puppet::Util::Puppetdb::CharEncoding.utf8_string({
+        :command => command,
+        :version => version,
+        :payload => payload,
+      # We use to_pson still here, to work around the support for shifting
+      # binary data from a catalog to PuppetDB. Attempting to use to_json
+      # we get to_json conversion errors:
+      #
+      #   Puppet source sequence is illegal/malformed utf-8
+      #   json/ext/GeneratorMethods.java:71:in `to_json'
+      #   puppet/util/puppetdb/command.rb:31:in `initialize'
+      #
+      # This is roughly inline with how Puppet serializes for catalogs as of
+      # Puppet 4.1.0. We need a better answer to non-utf8 data end-to-end.
+      }.to_pson, "Error encoding a '#{command}' command for host '#{certname}'")
+    end
+  end
+
+  attr_reader :command, :version, :certname, :payload
+
+  # Submit the command, returning the result hash.
+  #
+  # @return [Hash <String, String>]
+  def submit
+    checksum = Digest::SHA1.hexdigest(payload)
+
+    for_whom = " for #{certname}" if certname
+
+    begin
+      response = profile("Submit command HTTP post", [:puppetdb, :command, :submit]) do
+        Http.action("#{CommandsUrl}?checksum=#{checksum}") do |http_instance, path|
+          http_instance.post(path, payload, headers)
+        end
+      end
+
+      Puppet::Util::Puppetdb.log_x_deprecation_header(response)
+
+      if response.is_a? Net::HTTPSuccess
+        result = JSON.parse(response.body)
+        Puppet.info "'#{command}' command#{for_whom} submitted to PuppetDB with UUID #{result['uuid']}"
+        result
+      else
+        # Newline characters cause an HTTP error, so strip them
+        error = "[#{response.code} #{response.message}] #{response.body.gsub(/[\r\n]/, '')}"
+        if config.soft_write_failure
+          Puppet.err "'#{command}'command#{for_whom} failed during submission to PuppetDB: #{error}"
+        else
+          raise Puppet::Error, error
+        end
+      end
+    rescue => e
+      if config.soft_write_failure
+        Puppet.err e.message
+      else
+        # TODO: Use new exception handling methods from Puppet 3.0 here as soon as
+        #  we are able to do so (can't call them yet w/o breaking backwards
+        #  compatibility.)  We should either be using a nested exception or calling
+        #  Puppet::Util::Logging#log_exception or #log_and_raise here; w/o them
+        #  we lose context as to where the original exception occurred.
+        if Puppet[:trace]
+          Puppet.err(e)
+          Puppet.err(e.backtrace)
+        end
+        raise Puppet::Util::Puppetdb::CommandSubmissionError.new(e.message, {:command => command, :for_whom => for_whom})
+      end
+    end
+  end
+
+  # @!group Private instance methods
+
+  # @api private
+  def headers
+    {
+      "Accept" => "application/json",
+      "Content-Type" => "application/json; charset=utf-8",
+    }
+  end
+
+  # @api private
+  def config
+    # Would prefer to pass this to the constructor or acquire it some other
+    # way besides this pseudo-global reference.
+    Puppet::Util::Puppetdb.config
+  end
+
+end

data/puppet/lib/puppet/util/puppetdb/command_names.rb

@@ -0,0 +1,8 @@
+module Puppet::Util::Puppetdb
+  module CommandNames
+    CommandReplaceCatalog   = "replace catalog"
+    CommandReplaceFacts     = "replace facts"
+    CommandDeactivateNode   = "deactivate node"
+    CommandStoreReport      = "store report"
+  end
+end

data/puppet/lib/puppet/util/puppetdb/config.rb

@@ -0,0 +1,148 @@
+require 'puppet/util/puppetdb/command_names'
+require 'uri'
+
+module Puppet::Util::Puppetdb
+  class Config
+    include Puppet::Util::Puppetdb::CommandNames
+
+    # Public class methods
+
+    def self.load(config_file = nil)
+      defaults = {
+        :server                    => "puppetdb",
+        :port                      => 8081,
+        :soft_write_failure        => false,
+        :server_url_timeout        => 30,
+        :include_unchanged_resources => false,
+      }
+
+      config_file ||= File.join(Puppet[:confdir], "puppetdb.conf")
+
+      if File.exists?(config_file)
+        Puppet.debug("Configuring PuppetDB terminuses with config file #{config_file}")
+        content = File.read(config_file)
+      else
+        Puppet.debug("No #{config_file} file found; falling back to default server and port #{defaults[:server]}:#{defaults[:port]}")
+        content = ''
+      end
+
+      result = {}
+      section = nil
+      content.lines.each_with_index do |line,number|
+        # Gotta track the line numbers properly
+        number += 1
+        case line
+        when /^\[(\w+)\s*\]$/
+          section = $1
+          result[section] ||= {}
+
+        when /^\s*(\w+)\s*=\s*(\S+|[\S+\s*\,\s*\S]+)\s*$/
+          raise "Setting '#{line}' is illegal outside of section in PuppetDB config #{config_file}:#{number}" unless section
+          result[section][$1] = $2
+        when /^\s*[#;]/
+          # Skip comments
+        when /^\s*$/
+          # Skip blank lines
+        else
+          raise "Unparseable line '#{line}' in PuppetDB config #{config_file}:#{number}"
+        end
+      end
+
+      main_section = result['main'] || {}
+      # symbolize the keys
+      main_section = main_section.inject({}) {|h, (k,v)| h[k.to_sym] = v ; h}
+      # merge with defaults but filter out anything except the legal settings
+      config_hash = defaults.merge(main_section).reject do |k, v|
+        !([:server,
+           :port,
+           :ignore_blacklisted_events,
+           :include_unchanged_resources,
+           :soft_write_failure,
+           :server_urls,
+           :server_url_timeout].include?(k))
+      end
+
+      if config_hash[:server_urls]
+        uses_server_urls = true
+        config_hash[:server_urls] = config_hash[:server_urls].split(",").map {|s| s.strip}
+      else
+        uses_server_urls = false
+        config_hash[:server_urls] = ["https://#{config_hash[:server].strip}:#{config_hash[:port].to_s}"]
+      end
+      config_hash[:server_urls] = convert_and_validate_urls(config_hash[:server_urls])
+
+      config_hash[:server_url_timeout] = config_hash[:server_url_timeout].to_i
+      config_hash[:include_unchanged_resources] = Puppet::Util::Puppetdb.to_bool(config_hash[:include_unchanged_resources])
+      config_hash[:soft_write_failure] = Puppet::Util::Puppetdb.to_bool(config_hash[:soft_write_failure])
+
+      self.new(config_hash, uses_server_urls)
+    rescue => detail
+      Puppet.warning "Could not configure PuppetDB terminuses: #{detail}"
+      Puppet.warning detail.backtrace if Puppet[:trace]
+      raise
+    end
+
+    # @!group Public instance methods
+
+    def initialize(config_hash = {}, uses_server_urls=nil)
+      @config = config_hash
+      if !uses_server_urls
+        Puppet.warning("Specification of server and port in puppetdb.conf is deprecated. Use the setting server_urls.")
+      end
+      # To provide accurate error messages to users about HTTP failures, we
+      # need to know whether they initially defined their config via the old
+      # server/port combo or the new server_urls. This boolean keeps track
+      # of how the user defined that config so that we can give them a
+      # better error message
+      @server_url_config = uses_server_urls
+    end
+
+    def server_url_config?
+      @server_url_config
+    end
+
+    def server_urls
+      config[:server_urls]
+    end
+
+    def server_url_timeout
+      config[:server_url_timeout]
+    end
+
+    def include_unchanged_resources?
+      config[:include_unchanged_resources]
+    end
+
+    def soft_write_failure
+      config[:soft_write_failure]
+    end
+
+    # @!group Private instance methods
+
+    # @!attribute [r] count
+    #   @api private
+    attr_reader :config
+
+    # @api private
+    def self.convert_and_validate_urls(uri_strings)
+      uri_strings.map do |uri_string|
+
+        begin
+          uri = URI(uri_string.strip)
+        rescue URI::InvalidURIError => e
+          raise URI::InvalidURIError.new, "Error parsing URL '#{uri_string}' in PuppetDB 'server_urls', error message was '#{e.message}'"
+        end
+
+        if uri.scheme != 'https'
+          raise "PuppetDB 'server_urls' must be https, found '#{uri_string}'"
+        end
+
+        if uri.path != '' && uri.path != '/'
+          raise "PuppetDB 'server_urls' cannot contain URL paths, found '#{uri_string}'"
+        end
+        uri.path = ''
+        uri
+      end
+    end
+  end
+end