quixoten-puppetdb-terminus 2.3.8 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d309a703f223e7e3b3617452a21b6fd2dcd5bd38
-  data.tar.gz: b6b55228059a0551498171277d36316678b13fb8
+  metadata.gz: fd5513b1969cbbc8de118e435e8882b9d3b6f9c1
+  data.tar.gz: 016c79651a57659cbbc308dd0d4df9fd309f63eb
 SHA512:
-  metadata.gz: 056e58b8250c55111b0422f501e77efbeb979f9238decd035d58ac7ad41b9bd82d88e3b322ba627b34aab47abd394aba2fe09fc8ec8d88a60ae5baa39100a62b
-  data.tar.gz: 5216061084c04bb2f9beacd6d89bea1e98a13725f64935535198f82563a4924e9fd667067563fab7900dc5104549e56bd2d6f4dd2ab82c884dad27fc8b3255bd
+  metadata.gz: 38044e3513401f5b9a6361329d3e0856a2dea01e34bfb9965284befeed31f37dd32200ad7d7a82a0530c9d376fbc1139524dd8a8509913a266d9749cffcd3fce
+  data.tar.gz: 0b9505971f2583012847afd93d92f766b2221e3218d7724192d18a937d1bc2a36ad33f39cbc071f7f5a8760d81f6b5dfa539c385e9de94c9753ab47a04b875e7

data/lib/puppet/face/node/status.rb

@@ -17,14 +17,11 @@ Puppet::Face.define(:node, '0.0.1') do
       opts = args.pop
       raise ArgumentError, "Please provide at least one node" if args.empty?
 
-      server = Puppet::Util::Puppetdb.server
-      port = Puppet::Util::Puppetdb.port
-
-      http = Puppet::Network::HttpPool.http_instance(server, port)
-
       args.map do |node|
         begin
-          response = http.get(Puppet::Util::Puppetdb.url_path("/v3/nodes/#{CGI.escape(node)}"), headers)
+          response = Puppet::Util::Puppetdb::Http.action("/pdb/query/v4/nodes/#{CGI.escape(node)}") do |http_instance, path|
+             http_instance.get(path, headers)
+          end
           if response.is_a? Net::HTTPSuccess
             result = JSON.parse(response.body)
           elsif response.is_a? Net::HTTPNotFound
@@ -51,6 +48,8 @@ Puppet::Face.define(:node, '0.0.1') do
 
         if status['deactivated']
           lines << "Deactivated at #{status['deactivated']}"
+        elsif status['expired']
+          lines << "Expired at #{status['expired']}"
         else
           lines << "Currently active"
         end

data/lib/puppet/face/storeconfigs.rb

@@ -35,36 +35,36 @@ if Puppet::Util::Puppetdb.puppet3compat?
         begin
           Puppet::Rails.connect
 
+          timestamp = Time.now
+
           # Fetch all nodes, including exported resources and their params
           nodes = Puppet::Rails::Host.all(:include => {:resources => [:param_values, :puppet_tags]},
                                           :conditions => {:resources => {:exported => true}})
 
-          catalogs = nodes.map { |node| node_to_catalog_hash(node) }
+          catalogs = nodes.map { |node| node_to_catalog_hash(node, timestamp.iso8601(5)) }
 
           catalog_dir = File.join(workdir, 'catalogs')
           FileUtils.mkdir(catalog_dir)
 
           catalogs.each do |catalog|
-            filename = File.join(catalog_dir, "#{catalog[:data][:name]}.json")
+            filename = File.join(catalog_dir, "#{catalog[:certname]}.json")
 
             File.open(filename, 'w') do |file|
-              file.puts catalog.to_pson
+              file.puts catalog.to_json
             end
           end
 
           node_names = nodes.map(&:name).sort
 
-          timestamp = Time.now
-
           File.open(File.join(workdir, 'export-metadata.json'), 'w') do |file|
             metadata = {
               'timestamp' => timestamp,
-              'command-versions' => {
-                'replace-catalog' => 2,
+              'command_versions' => {
+                'replace_catalog' => 6,
               }
             }
 
-            file.puts metadata.to_pson
+            file.puts metadata.to_json
           end
 
           tarfile = destination_file(timestamp)
@@ -116,20 +116,21 @@ if Puppet::Util::Puppetdb.puppet3compat?
       Puppet::Util::Execution.execute(command)
     end
 
-    def node_to_catalog_hash(node)
+    def node_to_catalog_hash(node, timestamp)
       resources = node.resources.map { |resource| resource_to_hash(resource) }
       edges = node.resources.map { |resource| resource_to_edge_hash(resource) }
 
       {
+        :environment => "production",
         :metadata => {
           :api_version => 1,
         },
-        :data => {
-          :name => node.name,
-          :version => node.last_compile || Time.now,
-          :edges => edges,
-          :resources => resources + [stage_main_hash],
-        },
+        :certname => node.name,
+        :version => node.last_compile || Time.now,
+        :edges => edges,
+        :resources => resources + [stage_main_hash],
+        :timestamp => timestamp,
+        :producer_timestamp => timestamp,
       }
     end
 

data/lib/puppet/indirector/catalog/puppetdb.rb

@@ -10,7 +10,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
   def save(request)
     profile("catalog#save", [:puppetdb, :catalog, :save, request.key]) do
       catalog = munge_catalog(request.instance, extract_extra_request_data(request))
-      submit_command(request.key, catalog, CommandReplaceCatalog, 5)
+      submit_command(request.key, catalog, CommandReplaceCatalog, 6)
     end
   end
 
@@ -22,11 +22,16 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
   def extract_extra_request_data(request)
     {
       :transaction_uuid => request.options[:transaction_uuid],
-      :environment => request.environment,
-      :producer_timestamp => request.options[:producer_timestamp] || Time.now.iso8601,
+      :environment => request.environment.to_s,
+      :producer_timestamp => request.options[:producer_timestamp] || Time.now.iso8601(5),
     }
   end
 
+  def hashify_tags(hash)
+    hash["resources"] = hash["resources"].map { |resource| resource["tags"] = resource["tags"].to_data_hash; resource }
+    hash
+  end
+
   def munge_catalog(catalog, extra_request_data = {})
     profile("Munge catalog", [:puppetdb, :catalog, :munge]) do
       data = profile("Convert catalog to JSON data hash", [:puppetdb, :catalog, :convert_to_hash]) do
@@ -37,6 +42,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
       add_namevar_aliases(data, catalog)
       stringify_titles(data)
       stringify_version(data)
+      hashify_tags(data)
       sort_unordered_metaparams(data)
       munge_edges(data)
       synthesize_edges(data, catalog)
@@ -44,6 +50,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
       add_transaction_uuid(data, extra_request_data[:transaction_uuid])
       add_environment(data, extra_request_data[:environment])
       add_producer_timestamp(data, extra_request_data[:producer_timestamp])
+      change_name_to_certname(data)
 
       data
     end
@@ -60,6 +67,17 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
   # fundamentally unordered
   UnorderedMetaparams = [:alias, :audit, :before, :check, :notify, :require, :subscribe, :tag]
 
+  # Change the name field to certname
+  #
+  # @param hash [Hash] original data hash
+  # @return [Hash] returns original hash with 'name' changed to 'certname'
+  # @api private
+  def change_name_to_certname(hash)
+    hash['certname'] = hash.delete('name')
+
+    hash
+  end
+
   # Include environment in hash, returning the complete hash.
   #
   # @param hash [Hash] original data hash
@@ -79,7 +97,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
   # @return [Hash] returns original hash augmented with producer_timestamp
   # @api private
   def add_producer_timestamp(hash, producer_timestamp)
-    hash['producer-timestamp'] = producer_timestamp
+    hash['producer_timestamp'] = producer_timestamp
 
     hash
   end
@@ -91,7 +109,7 @@ class Puppet::Resource::Catalog::Puppetdb < Puppet::Indirector::REST
   # @return [Hash] returns original hash augmented with transaction_uuid
   # @api private
   def add_transaction_uuid(hash, transaction_uuid)
-    hash['transaction-uuid'] = transaction_uuid
+    hash['transaction_uuid'] = transaction_uuid
 
     hash
   end

data/lib/puppet/indirector/facts/puppetdb.rb

@@ -16,6 +16,14 @@ class Puppet::Node::Facts::Puppetdb < Puppet::Indirector::REST
     trusted.to_h
   end
 
+  def maybe_strip_internal(facts)
+    if Puppet::Node::Facts.method_defined? :strip_internal
+      facts.strip_internal
+    else
+      facts.values
+    end
+  end
+
   def save(request)
     profile("facts#save", [:puppetdb, :facts, :save, request.key]) do
       payload = profile("Encode facts command submission payload",
@@ -27,27 +35,28 @@ class Puppet::Node::Facts::Puppetdb < Puppet::Indirector::REST
           facts.values[:trusted] = get_trusted_info(request.node)
         end
         {
-          "name" => facts.name,
+          "certname" => facts.name,
           "values" => facts.values,
           # PDB-453: we call to_s to avoid a 'stack level too deep' error
           # when we attempt to use ActiveSupport 2.3.16 on RHEL 5 with
           # legacy storeconfigs.
           "environment" => request.options[:environment] || request.environment.to_s,
-          "producer-timestamp" => request.options[:producer_timestamp] || Time.now.iso8601,
+          "producer_timestamp" => request.options[:producer_timestamp] || Time.now.iso8601(5),
         }
       end
 
-      submit_command(request.key, payload, CommandReplaceFacts, 3)
+      submit_command(request.key, payload, CommandReplaceFacts, 4)
     end
   end
 
   def find(request)
     profile("facts#find", [:puppetdb, :facts, :find, request.key]) do
       begin
-        url = Puppet::Util::Puppetdb.url_path("/v3/nodes/#{CGI.escape(request.key)}/facts")
-        response = profile("Query for nodes facts: #{url}",
-                           [:puppetdb, :facts, :find, :query_nodes, request.key]) do
-          http_get(request, url, headers)
+        response = Http.action("/pdb/query/v4/nodes/#{CGI.escape(request.key)}/facts") do |http_instance, path|
+          profile("Query for nodes facts: #{URI.unescape(path)}",
+                  [:puppetdb, :facts, :find, :query_nodes, request.key]) do
+            http_instance.get(path, headers)
+          end
         end
         log_x_deprecation_header(response)
 
@@ -55,24 +64,20 @@ class Puppet::Node::Facts::Puppetdb < Puppet::Indirector::REST
           profile("Parse fact query response (size: #{response.body.size})",
                   [:puppetdb, :facts, :find, :parse_response, request.key]) do
             result = JSON.parse(response.body)
-            # Note: the Inventory Service API appears to expect us to return nil here
-            # if the node isn't found.  However, PuppetDB returns an empty array in
-            # this case; for now we will just look for that condition and assume that
-            # it means that the node wasn't found, so we will return nil.  In the
-            # future we may want to improve the logic such that we can distinguish
-            # between the "node not found" and the "no facts for this node" cases.
-            if result.empty?
-              return nil
-            end
+
             facts = result.inject({}) do |a,h|
               a.merge(h['name'] => h['value'])
             end
+
             Puppet::Node::Facts.new(request.key, facts)
           end
         else
           # Newline characters cause an HTTP error, so strip them
           raise "[#{response.code} #{response.message}] #{response.body.gsub(/[\r\n]/, '')}"
         end
+      rescue NotFoundError => e
+        # This is what the inventory service expects when there is no data
+        return nil
       rescue => e
         raise Puppet::Error, "Failed to find facts from PuppetDB at #{self.class.server}:#{self.class.port}: #{e}"
       end
@@ -115,10 +120,11 @@ class Puppet::Node::Facts::Puppetdb < Puppet::Indirector::REST
       query_param = CGI.escape(query.to_json)
 
       begin
-        url = Puppet::Util::Puppetdb.url_path("/v3/nodes?query=#{query_param}")
-        response = profile("Fact query request: #{URI.unescape(url)}",
-                           [:puppetdb, :facts, :search, :query_request, request.key]) do
-          http_get(request, url, headers)
+        response = Http.action("/pdb/query/v4/nodes?query=#{query_param}") do |http_instance, path|
+          profile("Fact query request: #{URI.unescape(path)}",
+                  [:puppetdb, :facts, :search, :query_request, request.key]) do
+            http_instance.get(path, headers)
+          end
         end
         log_x_deprecation_header(response)
 
@@ -132,7 +138,7 @@ class Puppet::Node::Facts::Puppetdb < Puppet::Indirector::REST
           raise "[#{response.code} #{response.message}] #{response.body.gsub(/[\r\n]/, '')}"
         end
       rescue => e
-        raise Puppet::Error, "Could not perform inventory search from PuppetDB at #{self.class.server}:#{self.class.port}: #{e}"
+        raise Puppet::Util::Puppetdb::InventorySearchError, e.message
       end
     end
   end

data/lib/puppet/indirector/node/puppetdb.rb

@@ -12,6 +12,8 @@ class Puppet::Node::Puppetdb < Puppet::Indirector::REST
   end
 
   def destroy(request)
-    submit_command(request.key, request.key, CommandDeactivateNode, 2)
+    payload = { :certname => request.key,
+                :producer_timestamp => request.options[:producer_timestamp] || Time.now.iso8601(5) }
+    submit_command(request.key, payload, CommandDeactivateNode, 3)
   end
 end

data/lib/puppet/indirector/resource/puppetdb.rb

@@ -27,11 +27,13 @@ class Puppet::Resource::Puppetdb < Puppet::Indirector::REST
       query_param = CGI.escape(expr.to_json)
 
       begin
-        url = Puppet::Util::Puppetdb.url_path("/v3/resources?query=#{query_param}")
-        response = profile("Resources query: #{URI.unescape(url)}",
-                           [:puppetdb, :resource, :search, :query, request.key]) do
-          http_get(request, url, headers)
+        response = Http.action("/pdb/query/v4/resources?query=#{query_param}") do |http_instance, path|
+          profile("Resources query: #{URI.unescape(path)}",
+                  [:puppetdb, :resource, :search, :query, request.key]) do
+            http_instance.get(path, headers)
+          end
         end
+
         log_x_deprecation_header(response)
 
         unless response.is_a? Net::HTTPSuccess

data/lib/puppet/reports/puppetdb.rb

@@ -19,7 +19,7 @@ Puppet::Reports.register_report(:puppetdb) do
   # @return [void]
   def process
     profile("report#process", [:puppetdb, :report, :process]) do
-      submit_command(self.host, report_to_hash, CommandStoreReport, 4)
+      submit_command(self.host, report_to_hash, CommandStoreReport, 5)
     end
 
     nil
@@ -37,17 +37,24 @@ Puppet::Reports.register_report(:puppetdb) do
         raise Puppet::Error, "Environment is nil, unable to submit report. This may be due a bug with Puppet. Ensure you are running the latest revision, see PUP-2508 for more details."
       end
 
+      resource_events = build_events_list
+      is_noop = resource_events.any? {|rs| rs["status"] == 'noop'} && resource_events.none? {|rs| rs["status"] == 'failed'}
+
       {
         "certname"                => host,
-        "puppet-version"          => puppet_version,
-        "report-format"           => report_format,
-        "configuration-version"   => configuration_version.to_s,
-        "start-time"              => Puppet::Util::Puppetdb.to_wire_time(time),
-        "end-time"                => Puppet::Util::Puppetdb.to_wire_time(time + run_duration),
-        "resource-events"         => build_events_list,
+        "puppet_version"          => puppet_version,
+        "report_format"           => report_format,
+        "configuration_version"   => configuration_version.to_s,
+        "producer_timestamp"      => Puppet::Util::Puppetdb.to_wire_time(Time.now),
+        "start_time"              => Puppet::Util::Puppetdb.to_wire_time(time),
+        "end_time"                => Puppet::Util::Puppetdb.to_wire_time(time + run_duration),
+        "resource_events"         => resource_events,
         "environment"             => environment,
-        "transaction-uuid"        => transaction_uuid,
+        "transaction_uuid"        => transaction_uuid,
         "status"                  => status,
+        "noop"                    => is_noop,
+        "logs"                    => build_logs_list,
+        "metrics"                 => build_metrics_list,
       }
     end
   end
@@ -69,6 +76,40 @@ Puppet::Reports.register_report(:puppetdb) do
     end
   end
 
+  # @return Array[Hash]
+  # @api private
+  def build_logs_list
+    profile("Build logs list (count: #{logs.count})",
+            [:puppetdb, :logs_list, :build]) do
+      logs.map do |log|
+        {
+          'file' => log.file,
+          'line' => log.line,
+          'level' => log.level,
+          'message' => log.message,
+          'source' => log.source,
+          'tags' => [*log.tags],
+          'time' => Puppet::Util::Puppetdb.to_wire_time(log.time),
+        }
+      end
+    end
+  end
+
+  # @return Array[Hash}
+  # @api private
+  def build_metrics_list
+    profile("Build metrics list (count: #{metrics.count})",
+            [:puppetdb, :metrics_list, :build]) do
+      metrics_list = []
+      metrics.each do |name, data|
+        metric_hashes = data.values.map {|x| {"category" => data.name, "name" => x.first, "value" => x.last}}
+        metrics_list.concat(metric_hashes)
+      end
+      metrics_list
+    end
+  end
+
+
   # @return Number
   # @api private
   def run_duration
@@ -97,15 +138,15 @@ Puppet::Reports.register_report(:puppetdb) do
     {
       "status"            => event.status,
       "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(event.time),
-      "resource-type"     => resource_status.resource_type,
-      "resource-title"    => resource_status.title.to_s,
+      "resource_type"     => resource_status.resource_type,
+      "resource_title"    => resource_status.title.to_s,
       "property"          => event.property,
-      "new-value"         => event.desired_value,
-      "old-value"         => event.previous_value,
+      "new_value"         => event.desired_value,
+      "old_value"         => event.previous_value,
       "message"           => event.message,
       "file"              => resource_status.file,
       "line"              => resource_status.line,
-      "containment-path"  => resource_status.containment_path,
+      "containment_path"  => resource_status.containment_path,
     }
   end
 
@@ -118,15 +159,15 @@ Puppet::Reports.register_report(:puppetdb) do
     {
       "status"            => event_status,
       "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(resource_status.time),
-      "resource-type"     => resource_status.resource_type,
-      "resource-title"    => resource_status.title.to_s,
+      "resource_type"     => resource_status.resource_type,
+      "resource_title"    => resource_status.title.to_s,
       "property"          => nil,
-      "new-value"         => nil,
-      "old-value"         => nil,
+      "new_value"         => nil,
+      "old_value"         => nil,
       "message"           => nil,
       "file"              => resource_status.file,
       "line"              => resource_status.line,
-      "containment-path" => resource_status.containment_path,
+      "containment_path" => resource_status.containment_path,
     }
   end
 

data/lib/puppet/util/puppetdb.rb

@@ -10,21 +10,19 @@ require 'fileutils'
 
 module Puppet::Util::Puppetdb
 
-  def self.server
-    config.server
+  class CommandSubmissionError < Puppet::Error
+    def initialize(msg, context)
+      super(msg)
+      @context = context
+    end
   end
 
-  def self.port
-    config.port
+  class InventorySearchError < Puppet::Error
   end
-
-  def self.url_path(path)
-    unless path.start_with?("/")
-      path = "/" + path
-    end
-    config.url_prefix + path
+  class SoftWriteFailError < Puppet::Error
   end
 
+
   def self.config
     @config ||= Puppet::Util::Puppetdb::Config.load
     @config
@@ -34,23 +32,6 @@ module Puppet::Util::Puppetdb
     defined?(Puppet::Parser::AST::HashOrArrayAccess)
   end
 
-  # This magical stuff is needed so that the indirector termini will make requests to
-  # the correct host/port, because this module gets mixed in to our indirector
-  # termini.
-  module ClassMethods
-    def server
-      Puppet::Util::Puppetdb.server
-    end
-
-    def port
-      Puppet::Util::Puppetdb.port
-    end
-  end
-
-  def self.included(child)
-    child.extend ClassMethods
-  end
-
   # Given an instance of ruby's Time class, this method converts it to a String
   # that conforms to PuppetDB's wire format for representing a date/time.
   def self.to_wire_time(time)
@@ -74,7 +55,7 @@ module Puppet::Util::Puppetdb
 
   # Submit a command to PuppetDB.
   #
-  # @param certname [String] hostname name of puppetdb instance
+  # @param certname [String] The certname this command operates on
   # @param payload [String] payload
   # @param command_name [String] name of command
   # @param version [Number] version number of command

data/lib/puppet/util/puppetdb/blacklist.rb

@@ -26,8 +26,8 @@ module Puppet::Util::Puppetdb
     end
 
     def is_event_blacklisted?(event)
-      @events.fetch(event["resource-type"], {}).
-        fetch(event["resource-title"], {}).
+      @events.fetch(event["resource_type"], {}).
+        fetch(event["resource_title"], {}).
         fetch(event["status"], {}).
         fetch(event["property"], false)
     end

data/lib/puppet/util/puppetdb/char_encoding.rb

@@ -33,7 +33,7 @@ module CharEncoding
 
 
   def self.utf8_string(str)
-    if RUBY_VERSION =~ /1.8/
+    if RUBY_VERSION =~ /^1.8/
       # Ruby 1.8 doesn't have String#encode and related methods, and there
       #  appears to be a bug in iconv that will interpret some byte sequences
       #  as 6-byte characters.  Thus, we are forced to resort to some unfortunate

data/lib/puppet/util/puppetdb/command.rb

@@ -1,6 +1,6 @@
 require 'puppet/error'
-require 'puppet/network/http_pool'
 require 'puppet/util/puppetdb'
+require 'puppet/util/puppetdb/http'
 require 'puppet/util/puppetdb/command_names'
 require 'puppet/util/puppetdb/char_encoding'
 require 'json'
@@ -9,7 +9,7 @@ class Puppet::Util::Puppetdb::Command
   include Puppet::Util::Puppetdb
   include Puppet::Util::Puppetdb::CommandNames
 
-  CommandsUrl = "/v3/commands"
+  CommandsUrl = "/pdb/cmd/v1"
 
   # Public instance methods
 
@@ -28,7 +28,21 @@ class Puppet::Util::Puppetdb::Command
     @version = version
     @certname = certname
     profile("Format payload", [:puppetdb, :payload, :format]) do
-      @payload = self.class.format_payload(command, version, payload)
+      @payload = Puppet::Util::Puppetdb::CharEncoding.utf8_string({
+        :command => command,
+        :version => version,
+        :payload => payload,
+      # We use to_pson still here, to work around the support for shifting
+      # binary data from a catalog to PuppetDB. Attempting to use to_json
+      # we get to_json conversion errors:
+      #
+      #   Puppet source sequence is illegal/malformed utf-8
+      #   json/ext/GeneratorMethods.java:71:in `to_json'
+      #   puppet/util/puppetdb/command.rb:31:in `initialize'
+      #
+      # This is roughly inline with how Puppet serializes for catalogs as of
+      # Puppet 4.1.0. We need a better answer to non-utf8 data end-to-end.
+      }.to_pson)
     end
   end
 
@@ -44,9 +58,9 @@ class Puppet::Util::Puppetdb::Command
 
     begin
       response = profile("Submit command HTTP post", [:puppetdb, :command, :submit]) do
-        http = Puppet::Network::HttpPool.http_instance(config.server, config.port)
-        http.post(Puppet::Util::Puppetdb.url_path(CommandsUrl + "?checksum=#{checksum}"),
-                  payload, headers)
+        Http.action("#{CommandsUrl}?checksum=#{checksum}") do |http_instance, path|
+          http_instance.post(path, payload, headers)
+        end
       end
 
       Puppet::Util::Puppetdb.log_x_deprecation_header(response)
@@ -65,42 +79,30 @@ class Puppet::Util::Puppetdb::Command
         end
       end
     rescue => e
-      error = "Failed to submit '#{command}' command#{for_whom} to PuppetDB at #{config.server}:#{config.port}: #{e}"
       if config.soft_write_failure
-        Puppet.err error
+        Puppet.err e.message
       else
         # TODO: Use new exception handling methods from Puppet 3.0 here as soon as
         #  we are able to do so (can't call them yet w/o breaking backwards
         #  compatibility.)  We should either be using a nested exception or calling
         #  Puppet::Util::Logging#log_exception or #log_and_raise here; w/o them
         #  we lose context as to where the original exception occurred.
-        puts e, e.backtrace if Puppet[:trace]
-        raise Puppet::Error, error
+        if Puppet[:trace]
+          Puppet.err(e)
+          Puppet.err(e.backtrace)
+        end
+        raise Puppet::Util::Puppetdb::CommandSubmissionError.new(e.message, {:command => command, :for_whom => for_whom})
       end
     end
   end
 
-
-  # @!group Private class methods
-
-  # @api private
-  def self.format_payload(command, version, payload)
-    message = {
-      :command => command,
-      :version => version,
-      :payload => payload,
-    }.to_pson
-
-    Puppet::Util::Puppetdb::CharEncoding.utf8_string(message)
-  end
-
   # @!group Private instance methods
 
   # @api private
   def headers
     {
       "Accept" => "application/json",
-      "Content-Type" => "application/json",
+      "Content-Type" => "application/json; charset=utf-8",
     }
   end
 

data/lib/puppet/util/puppetdb/config.rb

@@ -1,41 +1,43 @@
 require 'puppet/util/puppetdb/command_names'
 require 'puppet/util/puppetdb/blacklist'
+require 'uri'
 
 module Puppet::Util::Puppetdb
-class Config
-  include Puppet::Util::Puppetdb::CommandNames
-
-  # Public class methods
-
-  def self.load(config_file = nil)
-    defaults = {
-      :server                    => "puppetdb",
-      :port                      => 8081,
-      :url_prefix                => "",
-      :soft_write_failure        => false,
-      :ignore_blacklisted_events => true,
-    }
-
-    config_file ||= File.join(Puppet[:confdir], "puppetdb.conf")
-
-    if File.exists?(config_file)
-      Puppet.debug("Configuring PuppetDB terminuses with config file #{config_file}")
-      content = File.read(config_file)
-    else
-      Puppet.debug("No #{config_file} file found; falling back to default server and port #{defaults[:server]}:#{defaults[:port]}")
-      content = ''
-    end
+  class Config
+    include Puppet::Util::Puppetdb::CommandNames
+
+    # Public class methods
+
+    def self.load(config_file = nil)
+      defaults = {
+        :server                    => "puppetdb",
+        :port                      => 8081,
+        :soft_write_failure        => false,
+        :ignore_blacklisted_events => true,
+        :server_url_timeout        => 30
+      }
+
+      config_file ||= File.join(Puppet[:confdir], "puppetdb.conf")
+
+      if File.exists?(config_file)
+        Puppet.debug("Configuring PuppetDB terminuses with config file #{config_file}")
+        content = File.read(config_file)
+      else
+        Puppet.debug("No #{config_file} file found; falling back to default server and port #{defaults[:server]}:#{defaults[:port]}")
+        content = ''
+      end
 
-    result = {}
-    section = nil
-    content.lines.each_with_index do |line,number|
-      # Gotta track the line numbers properly
-      number += 1
-      case line
+      result = {}
+      section = nil
+      content.lines.each_with_index do |line,number|
+        # Gotta track the line numbers properly
+        number += 1
+        case line
         when /^\[(\w+)\s*\]$/
           section = $1
           result[section] ||= {}
-        when /^\s*(\w+)\s*=\s*(\S+)\s*$/
+
+        when /^\s*(\w+)\s*=\s*(\S+|[\S+\s*\,\s*\S]+)\s*$/
           raise "Setting '#{line}' is illegal outside of section in PuppetDB config #{config_file}:#{number}" unless section
           result[section][$1] = $2
         when /^\s*[#;]/
@@ -44,86 +46,115 @@ class Config
           # Skip blank lines
         else
           raise "Unparseable line '#{line}' in PuppetDB config #{config_file}:#{number}"
+        end
       end
-    end
 
+      main_section = result['main'] || {}
+      # symbolize the keys
+      main_section = main_section.inject({}) {|h, (k,v)| h[k.to_sym] = v ; h}
+      # merge with defaults but filter out anything except the legal settings
+      config_hash = defaults.merge(main_section).reject do |k, v|
+        !([:server,
+           :port,
+           :ignore_blacklisted_events,
+           :soft_write_failure,
+           :server_urls,
+           :server_url_timeout].include?(k))
+      end
 
-    main_section = result['main'] || {}
-    # symbolize the keys
-    main_section = main_section.inject({}) {|h, (k,v)| h[k.to_sym] = v ; h}
-    # merge with defaults but filter out anything except the legal settings
-    config_hash = defaults.merge(main_section).reject do |k, v|
-      !([:server, :port, :url_prefix, :ignore_blacklisted_events, :soft_write_failure].include?(k))
+      if config_hash[:server_urls]
+        uses_server_urls = true
+        config_hash[:server_urls] = config_hash[:server_urls].split(",").map {|s| s.strip}
+      else
+        uses_server_urls = false
+        config_hash[:server_urls] = ["https://#{config_hash[:server].strip}:#{config_hash[:port].to_s}"]
+      end
+      config_hash[:server_urls] = convert_and_validate_urls(config_hash[:server_urls])
+      
+      config_hash[:server_url_timeout] = config_hash[:server_url_timeout].to_i
+      config_hash[:ignore_blacklisted_events] = Puppet::Util::Puppetdb.to_bool(config_hash[:ignore_blacklisted_events])
+      config_hash[:soft_write_failure] = Puppet::Util::Puppetdb.to_bool(config_hash[:soft_write_failure])
+
+      self.new(config_hash, uses_server_urls)
+    rescue => detail
+      Puppet.warning "Could not configure PuppetDB terminuses: #{detail}"
+      Puppet.warning detail.backtrace if Puppet[:trace]
+      raise
     end
 
-    config_hash[:server] = config_hash[:server].strip
-    config_hash[:port] = config_hash[:port].to_i
-    config_hash[:url_prefix] = normalize_url_prefix(config_hash[:url_prefix].strip)
-    config_hash[:ignore_blacklisted_events] =
-      Puppet::Util::Puppetdb.to_bool(config_hash[:ignore_blacklisted_events])
-    config_hash[:soft_write_failure] =
-      Puppet::Util::Puppetdb.to_bool(config_hash[:soft_write_failure])
-
-    self.new(config_hash)
-  rescue => detail
-    puts detail.backtrace if Puppet[:trace]
-    Puppet.warning "Could not configure PuppetDB terminuses: #{detail}"
-    raise
-  end
-
-  # @!group Public instance methods
+    # @!group Public instance methods
 
-  def initialize(config_hash = {})
-    @config = config_hash
-    initialize_blacklisted_events()
-  end
-
-  def server
-    config[:server]
-  end
+    def initialize(config_hash = {}, uses_server_urls=nil)
+      @config = config_hash
+      initialize_blacklisted_events()
+      if !uses_server_urls
+        Puppet.warning("Specification of server and port in puppetdb.conf is deprecated. Use the setting server_urls.")
+      end
+      # To provide accurate error messages to users about HTTP failures, we
+      # need to know whether they initially defined their config via the old
+      # server/port combo or the new server_urls. This boolean keeps track
+      # of how the user defined that config so that we can give them a
+      # better error message
+      @server_url_config = uses_server_urls
+    end
 
-  def port
-    config[:port]
-  end
+    def server_url_config?
+      @server_url_config
+    end
 
-  def url_prefix
-    config[:url_prefix]
-  end
+    def server_urls
+      config[:server_urls]
+    end
 
-  def ignore_blacklisted_events?
-    config[:ignore_blacklisted_events]
-  end
+    def server_url_timeout
+      config[:server_url_timeout]
+    end
 
-  def is_event_blacklisted?(event)
-   @blacklist.is_event_blacklisted? event
-  end
+    def ignore_blacklisted_events?
+      config[:ignore_blacklisted_events]
+    end
 
-  def soft_write_failure
-    config[:soft_write_failure]
-  end
+    def is_event_blacklisted?(event)
+      @blacklist.is_event_blacklisted? event
+    end
 
-  # @!group Private class methods
-  def self.normalize_url_prefix(prefix)
-    if prefix == ""
-      prefix
-    elsif prefix.start_with?("/")
-      prefix
-    else
-      "/" + prefix
+    def soft_write_failure
+      config[:soft_write_failure]
     end
-  end
 
-  # @!group Private instance methods
+    # @!group Private instance methods
 
-  # @!attribute [r] count
-  #   @api private
-  attr_reader :config
+    # @!attribute [r] count
+    #   @api private
+    attr_reader :config
 
-  Blacklist = Puppet::Util::Puppetdb::Blacklist
+    Blacklist = Puppet::Util::Puppetdb::Blacklist
+
+    # @api private
+    def initialize_blacklisted_events(events = Blacklist::BlacklistedEvents)
+      @blacklist = Blacklist.new(events)
+    end
 
-  # @api private
-  def initialize_blacklisted_events(events = Blacklist::BlacklistedEvents)
-    @blacklist = Blacklist.new(events)
+    def self.convert_and_validate_urls(uri_strings)
+      uri_strings.map do |uri_string|
+
+        begin
+          uri = URI(uri_string.strip)
+        rescue URI::InvalidURIError => e
+          raise URI::InvalidURIError.new, "Error parsing URL '#{uri_string}' in PuppetDB 'server_urls', error message was '#{e.message}'"
+        end
+
+        if uri.scheme != 'https'
+          raise "PuppetDB 'server_urls' must be https, found '#{uri_string}'"
+        end
+
+        if uri.path != '' && uri.path != '/'
+          raise "PuppetDB 'server_urls' cannot contain URL paths, found '#{uri_string}'"
+        end
+        uri.path = ''
+        
+        uri
+      end
+    end
   end
 end
-end

data/lib/puppet/util/puppetdb/http.rb

@@ -0,0 +1,121 @@
+require 'uri'
+require 'puppet/network/http_pool'
+require 'net/http'
+require 'timeout'
+require 'pp'
+
+module Puppet::Util::Puppetdb
+  class Http
+
+    SERVER_URL_FAIL_MSG = "Failing over to the next PuppetDB url in the 'server_urls' list"
+
+    # Concat two url snippets, taking into account a trailing/leading slash to
+    # ensure a correct url is constructed
+    #
+    # @param snippet1 [String] first URL snippet
+    # @param snippet2 [String] second URL snippet
+    # @return [String] returns http response
+    # @api private
+    def self.concat_url_snippets(snippet1, snippet2)
+      if snippet1.end_with?('/') and snippet2.start_with?('/')
+        snippet1 + snippet2[1..-1]
+      elsif !snippet1.end_with?('/') and !snippet2.start_with?('/')
+        snippet1 + '/' + snippet2
+      else
+        snippet1 + snippet2
+      end
+    end
+
+    # Setup an http connection, provide a block that will do something with that http
+    # connection. The block should be a two argument block, accepting the connection (which
+    # you can call get or post on for example) and the properly constructed path, which
+    # will be the concatenated version of any url_prefix and the path passed in.
+    #
+    # @param path_suffix [String] path for the get/post of the http action
+    # @param http_callback [Proc] proc containing the code calling the action on the http connection
+    # @return [Response] returns http response
+    def self.action(path_suffix, &http_callback)
+
+      response = nil
+      config = Puppet::Util::Puppetdb.config
+      server_url_config = config.server_url_config?
+
+      for url in Puppet::Util::Puppetdb.config.server_urls
+        begin
+          route = concat_url_snippets(url.request_uri, path_suffix)
+          http = Puppet::Network::HttpPool.http_instance(url.host, url.port)
+          request_timeout = config.server_url_timeout
+
+          response = timeout(request_timeout) do
+            http_callback.call(http, route)
+          end
+
+          if response.is_a? Net::HTTPServerError
+            Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{response.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+            response = nil
+          elsif response.is_a? Net::HTTPNotFound
+            if response.body && response.body.chars.first == "{"
+              # If it appears to be json, we've probably gotten an authentic 'not found' message.
+              Puppet.debug("HTTP 404 (probably normal) when connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{response.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+              response = :notfound
+            else
+              # But we can also get 404s when conneting to a puppetdb that's still starting or due to misconfiguration.
+              Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{response.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+              response = nil
+            end
+          else
+            break
+          end
+        rescue Timeout::Error => e
+          Puppet.warning("Request to #{url.host} on #{url.port} at route #{route} timed out after #{request_timeout} seconds. #{SERVER_URL_FAIL_MSG if server_url_config}")
+
+        rescue SocketError, OpenSSL::SSL::SSLError, SystemCallError, Net::ProtocolError, IOError, Net::HTTPNotFound => e
+          Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{e.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+
+        rescue Puppet::Util::Puppetdb::InventorySearchError => e
+          Puppet.warning("Could not perform inventory search from PuppetDB at #{url.host}:#{url.port}: '#{e.message}' #{SERVER_URL_FAIL_MSG if server_url_config}")
+
+        rescue Puppet::Util::Puppetdb::CommandSubmissionError => e
+          error = "Failed to submit '#{e.context[:command]}' command for '#{e.context[:for_whom]}' to PuppetDB at #{url.host}:#{url.port}: '#{e.message}'."
+          if config.soft_write_failure
+            Puppet.err error
+          else
+            Puppet.warning(error + " #{SERVER_URL_FAIL_MSG if server_url_config}")
+          end
+        rescue Puppet::Util::Puppetdb::SoftWriteFailError => e
+          Puppet.warning("Failed to submit '#{e.context[:command]}' command for '#{e.context[:for_whom]}' to PuppetDB at #{url.host}:#{url.port}: '#{e.message}' #{SERVER_URL_FAIL_MSG if server_url_config}")
+        rescue Puppet::Error => e
+          if e.message =~ /did not match server certificate; expected one of/
+            Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{e.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+          else
+            raise
+          end
+        end
+      end
+
+      if response.nil? or response == :notfound
+        if server_url_config
+          server_url_strings = Puppet::Util::Puppetdb.config.server_urls.map {|url| url.to_s}.join(', ')
+          if response == :notfound
+            raise NotFoundError, "Failed to find '#{path_suffix}' on any of the following 'server_urls': #{server_url_strings}"
+          else
+            raise Puppet::Error, "Failed to execute '#{path_suffix}' on any of the following 'server_urls': #{server_url_strings}"
+          end
+        else
+          uri = Puppet::Util::Puppetdb.config.server_urls.first
+          if response == :notfound
+            raise NotFoundError, "Failed to find '#{path_suffix}' on server: '#{uri.host}' and port: '#{uri.port}'"
+          else
+            raise Puppet::Error, "Failed to execute '#{path_suffix}' on server: '#{uri.host}' and port: '#{uri.port}'"
+          end
+        end
+      end
+
+      response
+
+    end
+  end
+
+  class NotFoundError < Puppet::Error
+  end
+end

data/lib/puppetdb/terminus/version.rb

@@ -1,6 +1,6 @@
 module PuppetDB
   module Terminus
-    VERSION          = "2.3.8"
+    VERSION          = "3.0.0"
     UPSTREAM_VERSION = VERSION.split(".")[0..2].join(".")
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quixoten-puppetdb-terminus
 version: !ruby/object:Gem::Version
-  version: 2.3.8
+  version: 3.0.0
 platform: ruby
 authors:
 - Devin Christensen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-01 00:00:00.000000000 Z
+date: 2016-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,7 @@ files:
 - lib/puppet/util/puppetdb/command.rb
 - lib/puppet/util/puppetdb/command_names.rb
 - lib/puppet/util/puppetdb/config.rb
+- lib/puppet/util/puppetdb/http.rb
 - lib/puppetdb-terminus.rb
 - lib/puppetdb/terminus.rb
 - lib/puppetdb/terminus/version.rb
@@ -91,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: PuppetDB Terminus