quixoten-puppetdb-terminus 3.1.1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b5638341ba0d3ae5fe31ed83d385ef11f3ea7368
-  data.tar.gz: a84380223cf2cd971a11376e54f98bbc4815e363
+  metadata.gz: 616f4054f25a66e89ff21c6af1a5d2777210c25a
+  data.tar.gz: 35b9513e9f12f34bd6bee4fb02f5d29fc8aec0c1
 SHA512:
-  metadata.gz: 39986de0b1ce9704f30eb68f83e8a2d017f185eebe091e0c793ac7b1ce86ea364968eb2eddec46d572a5f8f44816d974d3fa0eac5c93ea4c500cd0a401977c54
-  data.tar.gz: c24beffee4862e5587357f5a3f5f602c97752e4e196895265be3e04408a9753d1461a91b76622abdf5b586feb170db9b6e6e4b04e45aabc033bba795727dcba3
+  metadata.gz: 52e43ec18885c215f8bd500a4682591aa57301c21d3ff2ca1313bb672f73f3441cd202abd1db0ac287e19f7d00918b6a046f14517fadc07eb0fdedf434c4d916
+  data.tar.gz: 6998777cc272a6cd011f96805f4dccd0d831ae2df3cd49967e420dc6aa8f27ab98b3a98dbcaee3b66d3c71c68610692d185d9ee5440d9d8ca7bd093fa303ab16

data/lib/puppet/indirector/catalog/puppetdb.rb

@@ -10,7 +10,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
   def save(request)
     profile("catalog#save", [:puppetdb, :catalog, :save, request.key]) do
       catalog = munge_catalog(request.instance, extract_extra_request_data(request))
-      submit_command(request.key, catalog, CommandReplaceCatalog, 6)
+      submit_command(request.key, catalog, CommandReplaceCatalog, 7)
     end
   end
 
@@ -24,6 +24,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
       :transaction_uuid => request.options[:transaction_uuid],
       :environment => request.environment.to_s,
       :producer_timestamp => request.options[:producer_timestamp] || Time.now.iso8601(5),
+      :code_id => request.options[:code_id],
     }
   end
 
@@ -51,6 +52,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
       add_environment(data, extra_request_data[:environment])
       add_producer_timestamp(data, extra_request_data[:producer_timestamp])
       change_name_to_certname(data)
+      add_code_id(data, extra_request_data[:code_id])
 
       data
     end
@@ -114,6 +116,18 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
     hash
   end
 
+  # Include code_id in hash, returning the complete hash.
+  #
+  # @param hash [Hash] original data hash
+  # @param code_id [String] code_id
+  # @return [Hash] returns original hash augmented with transaction_uuid
+  # @api private
+  def add_code_id(hash, code_id)
+    hash['code_id'] = code_id
+
+    hash
+  end
+
   # Version is an integer (time since epoch in millis). The wire
   # format specifies version should be a string
   #
@@ -200,7 +214,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
         params = resource['parameters']
         UnorderedMetaparams.each do |metaparam|
           if params[metaparam].kind_of? Array then
-            values = params[metaparam].sort
+            values = params[metaparam].sort_by {|x| x.to_s}
             params[metaparam] = values unless values.empty?
           end
         end

data/lib/puppet/util/puppetdb/char_encoding.rb

@@ -31,37 +31,141 @@ module CharEncoding
 
   Utf8ReplacementChar = [ 0xEF, 0xBF, 0xBD ].pack("c*")
 
+  DEFAULT_INVALID_CHAR = "\ufffd"
 
-  def self.utf8_string(str)
+  # @api private
+  def self.all_indexes_of_char(str, char)
+    (0..str.length).find_all{ |i| str[i] == char}
+  end
+
+  # @api private
+  #
+  # Takes an array and returns a sub-array without the last element
+  #
+  # @return [Object]
+  def self.drop_last(array)
+    array[0..-2]
+  end
+
+  # @api private
+  #
+  # Takes an array of increasing integers and collapses the sequential
+  # integers into ranges
+  #
+  # @param index_array an array of sorted integers
+  # @return [Range]
+  def self.collapse_ranges(index_array)
+    ranges = index_array.each.inject([]) do |spans, n|
+      if spans.empty? || spans.last.end != n - 1
+        spans << Range.new(n, n)
+      else
+        drop_last(spans) << Range.new(spans.last.begin,n)
+      end
+    end
+  end
+
+  # @api private
+  #
+  # Scans the string s with bad characters found at bad_char_indexes
+  # and returns an array of messages that give some context around the
+  # bad characters. This will give up to 100 characters prior to the
+  # bad character and 100 after. It will return fewer if it's at the
+  # beginning of a string or if another bad character appears before
+  # reaching the 100 characters
+  #
+  # @param str string coming from to_pson, likely a command to be submitted to PDB
+  # @param bad_char_indexes an array of indexes into the string where invalid characters were found
+  # @return [String]
+  def self.error_char_context(str, bad_char_indexes)
+    bad_char_ranges = collapse_ranges(bad_char_indexes)
+    bad_char_ranges.each_with_index.inject([]) do |state, (r, index)|
+      gap = r.to_a.length
+
+      prev_bad_char_end = bad_char_ranges[index-1].end + 1 if index > 0
+      next_bad_char_begin = bad_char_ranges[index+1].begin - 1 if index < bad_char_ranges.length - 1
+
+      start_char = [prev_bad_char_end || 0, r.begin-100].max
+      end_char = [next_bad_char_begin || str.length - 1, r.end+100].min
+      x = [next_bad_char_begin || str.length, r.end+100, str.length]
+      prefix = str[start_char..r.begin-1]
+      suffix = str[r.end+1..end_char]
+
+      state << "'#{prefix}' followed by #{gap} invalid/undefined bytes then '#{suffix}'"
+    end
+  end
+
+  # @api private
+  #
+  # Warns the user if an invalid character was found. If debugging is
+  # enabled will also log contextual information about where the bad
+  # character(s) were found
+  #
+  # @param str A string coming from to_pson, likely a command to be submitted to PDB
+  # @param error_context_str information about where this string came from for use in error messages
+  # @return String
+  def self.warn_if_invalid_chars(str, error_context_str)
+    bad_char_indexes = all_indexes_of_char(str, DEFAULT_INVALID_CHAR)
+    if bad_char_indexes.empty?
+      str
+    else
+      Puppet.warning "#{error_context_str} ignoring invalid UTF-8 byte sequences in data to be sent to PuppetDB, see debug logging for more info"
+      if Puppet.settings[:log_level] == "debug"
+        Puppet.debug error_context_str + "\n" + error_char_context(str, bad_char_indexes).join("\n")
+      end
+
+      str
+    end
+  end
+
+  # @api private
+  #
+  # Attempts to coerce str to UTF-8, if that fails will output context
+  # information using error_context_str
+  #
+  # @param str A string coming from to_pson, likely a command to be submitted to PDB
+  # @param error_context_str information about where this string came from for use in error messages
+  # @return Str
+  def self.coerce_to_utf8(str, error_context_str)
+    str_copy = str.dup
+    # This code is passed in a string that was created by
+    # to_pson. to_pson calls force_encoding('ASCII-8BIT') on the
+    # string before it returns it. This leaves the actual UTF-8 bytes
+    # alone. Below we check to see if this is the case (this should be
+    # most common). In this case, the bytes are still UTF-8 and we can
+    # just encode! and we're good to go. If They are not valid UTF-8
+    # bytes, that means there is probably some binary data mixed in
+    # the middle of the UTF-8 string. In this case we need to output a
+    # warning and give the user more information
+    str_copy.force_encoding("UTF-8")
+    if str_copy.valid_encoding?
+      str_copy.encode!("UTF-8")
+    else
+      # This is force_encoded as US-ASCII to avoid any overlapping
+      # byte related issues that could arise from mis-interpreting a
+      # random extra byte as part of a multi-byte UTF-8 character
+      str_copy.force_encoding("US-ASCII")
+      warn_if_invalid_chars(str_copy.encode!("UTF-8",
+                                             :invalid => :replace,
+                                             :undef => :replace,
+                                             :replace => DEFAULT_INVALID_CHAR),
+                            error_context_str)
+    end
+  end
+
+  def self.utf8_string(str, error_context_str)
     if RUBY_VERSION =~ /^1.8/
       # Ruby 1.8 doesn't have String#encode and related methods, and there
       #  appears to be a bug in iconv that will interpret some byte sequences
       #  as 6-byte characters.  Thus, we are forced to resort to some unfortunate
       #  manual chicanery.
       warn_if_changed(str, ruby18_clean_utf8(str))
-    elsif str.encoding == Encoding::UTF_8
-      # If we get here, we're in ruby 1.9+, so we have the string encoding methods
-      #  available.  However, just because  a ruby String object is already
-      #  marked as UTF-8, that doesn't guarantee that its contents are actually
-      #  valid; and if you call ruby's ".encode" method with an encoding of
-      #  "utf-8" for a String that ruby already believes is UTF-8, ruby
-      #  seems to optimize that to be a no-op.  So, we have to do some more
-      #  complex handling...
-
-      # If the string already has valid encoding then we're fine.
-      return str if str.valid_encoding?
-
-      # If not, we basically have to walk over the characters and replace
-      #  them by hand.
-      warn_if_changed(str, str.each_char.map { |c| c.valid_encoding? ? c : "\ufffd"}.join)
     else
-      # if we get here, we're ruby 1.9 and the current string is *not* encoded
-      #  as UTF-8.  Thus we can actually rely on ruby's "encode" method.
       begin
-        str.encode('UTF-8')
+        coerce_to_utf8(str, error_context_str)
       rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError => e
-        # If we got an exception, the string is either invalid or not
-        # convertible to UTF-8, so drop those bytes.
+          # If we got an exception, the string is either invalid or not
+          # convertible to UTF-8, so drop those bytes.
+
         warn_if_changed(str, str.encode('UTF-8', :invalid => :replace, :undef => :replace))
       end
     end

data/lib/puppet/util/puppetdb/command.rb

@@ -42,7 +42,7 @@ class Puppet::Util::Puppetdb::Command
       #
       # This is roughly inline with how Puppet serializes for catalogs as of
       # Puppet 4.1.0. We need a better answer to non-utf8 data end-to-end.
-      }.to_pson)
+      }.to_pson, "Error encoding a '#{command}' command for host '#{certname}'")
     end
   end
 

data/lib/puppetdb/terminus/version.rb

@@ -1,6 +1,6 @@
 module PuppetDB
   module Terminus
-    VERSION          = "3.1.1"
+    VERSION          = "3.2.0"
     UPSTREAM_VERSION = VERSION.split(".")[0..2].join(".")
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: quixoten-puppetdb-terminus
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.2.0
 platform: ruby
 authors:
 - Devin Christensen