ml-puppetdb-terminus 3.2.1

data/puppet/lib/puppet/util/puppetdb/http.rb

@@ -0,0 +1,121 @@
+require 'uri'
+require 'puppet/network/http_pool'
+require 'net/http'
+require 'timeout'
+require 'pp'
+
+module Puppet::Util::Puppetdb
+  class Http
+
+    SERVER_URL_FAIL_MSG = "Failing over to the next PuppetDB url in the 'server_urls' list"
+
+    # Concat two url snippets, taking into account a trailing/leading slash to
+    # ensure a correct url is constructed
+    #
+    # @param snippet1 [String] first URL snippet
+    # @param snippet2 [String] second URL snippet
+    # @return [String] returns http response
+    # @api private
+    def self.concat_url_snippets(snippet1, snippet2)
+      if snippet1.end_with?('/') and snippet2.start_with?('/')
+        snippet1 + snippet2[1..-1]
+      elsif !snippet1.end_with?('/') and !snippet2.start_with?('/')
+        snippet1 + '/' + snippet2
+      else
+        snippet1 + snippet2
+      end
+    end
+
+    # Setup an http connection, provide a block that will do something with that http
+    # connection. The block should be a two argument block, accepting the connection (which
+    # you can call get or post on for example) and the properly constructed path, which
+    # will be the concatenated version of any url_prefix and the path passed in.
+    #
+    # @param path_suffix [String] path for the get/post of the http action
+    # @param http_callback [Proc] proc containing the code calling the action on the http connection
+    # @return [Response] returns http response
+    def self.action(path_suffix, &http_callback)
+
+      response = nil
+      config = Puppet::Util::Puppetdb.config
+      server_url_config = config.server_url_config?
+
+      for url in Puppet::Util::Puppetdb.config.server_urls
+        begin
+          route = concat_url_snippets(url.request_uri, path_suffix)
+          http = Puppet::Network::HttpPool.http_instance(url.host, url.port)
+          request_timeout = config.server_url_timeout
+
+          response = timeout(request_timeout) do
+            http_callback.call(http, route)
+          end
+
+          if response.is_a? Net::HTTPServerError
+            Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{response.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+            response = nil
+          elsif response.is_a? Net::HTTPNotFound
+            if response.body && response.body.chars.first == "{"
+              # If it appears to be json, we've probably gotten an authentic 'not found' message.
+              Puppet.debug("HTTP 404 (probably normal) when connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{response.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+              response = :notfound
+            else
+              # But we can also get 404s when conneting to a puppetdb that's still starting or due to misconfiguration.
+              Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{response.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+              response = nil
+            end
+          else
+            break
+          end
+        rescue Timeout::Error => e
+          Puppet.warning("Request to #{url.host} on #{url.port} at route #{route} timed out after #{request_timeout} seconds. #{SERVER_URL_FAIL_MSG if server_url_config}")
+
+        rescue SocketError, OpenSSL::SSL::SSLError, SystemCallError, Net::ProtocolError, IOError, Net::HTTPNotFound => e
+          Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{e.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+
+        rescue Puppet::Util::Puppetdb::InventorySearchError => e
+          Puppet.warning("Could not perform inventory search from PuppetDB at #{url.host}:#{url.port}: '#{e.message}' #{SERVER_URL_FAIL_MSG if server_url_config}")
+
+        rescue Puppet::Util::Puppetdb::CommandSubmissionError => e
+          error = "Failed to submit '#{e.context[:command]}' command for '#{e.context[:for_whom]}' to PuppetDB at #{url.host}:#{url.port}: '#{e.message}'."
+          if config.soft_write_failure
+            Puppet.err error
+          else
+            Puppet.warning(error + " #{SERVER_URL_FAIL_MSG if server_url_config}")
+          end
+        rescue Puppet::Util::Puppetdb::SoftWriteFailError => e
+          Puppet.warning("Failed to submit '#{e.context[:command]}' command for '#{e.context[:for_whom]}' to PuppetDB at #{url.host}:#{url.port}: '#{e.message}' #{SERVER_URL_FAIL_MSG if server_url_config}")
+        rescue Puppet::Error => e
+          if e.message =~ /did not match server certificate; expected one of/
+            Puppet.warning("Error connecting to #{url.host} on #{url.port} at route #{route}, error message received was '#{e.message}'. #{SERVER_URL_FAIL_MSG if server_url_config}")
+          else
+            raise
+          end
+        end
+      end
+
+      if response.nil? or response == :notfound
+        if server_url_config
+          server_url_strings = Puppet::Util::Puppetdb.config.server_urls.map {|url| url.to_s}.join(', ')
+          if response == :notfound
+            raise NotFoundError, "Failed to find '#{path_suffix}' on any of the following 'server_urls': #{server_url_strings}"
+          else
+            raise Puppet::Error, "Failed to execute '#{path_suffix}' on any of the following 'server_urls': #{server_url_strings}"
+          end
+        else
+          uri = Puppet::Util::Puppetdb.config.server_urls.first
+          if response == :notfound
+            raise NotFoundError, "Failed to find '#{path_suffix}' on server: '#{uri.host}' and port: '#{uri.port}'"
+          else
+            raise Puppet::Error, "Failed to execute '#{path_suffix}' on server: '#{uri.host}' and port: '#{uri.port}'"
+          end
+        end
+      end
+
+      response
+
+    end
+  end
+
+  class NotFoundError < Puppet::Error
+  end
+end

data/puppet/spec/README.markdown

@@ -0,0 +1,8 @@
+NOTE
+====
+
+This project's specs depend on puppet core, and thus they require the
+puppetlabs_spec_helper project.  For more information please see the
+README in that project, which can be found here:
+
+https://github.com/puppetlabs/puppetlabs_spec_helper

data/puppet/spec/spec.opts

@@ -0,0 +1,6 @@
+--format
+s
+--colour
+--loadby
+mtime
+--backtrace

data/puppet/spec/spec_helper.rb

@@ -0,0 +1,38 @@
+dir = File.expand_path(File.dirname(__FILE__))
+$LOAD_PATH.unshift File.join(dir, "../lib")
+# Maybe puppetlabs_spec_helper is in a directory next to puppetdb. If not, we
+# don't fail any worse than we already would.
+$LOAD_PATH.push File.join(dir, "../../../puppetlabs_spec_helper")
+
+require 'rspec'
+require 'rspec/expectations'
+require 'puppetlabs_spec_helper/puppet_spec_helper'
+require 'tmpdir'
+require 'fileutils'
+require 'puppet'
+require 'puppet/util/puppetdb'
+require 'puppet/util/log'
+
+def create_environmentdir(environment)
+  if not Puppet::Util::Puppetdb.puppet3compat?
+    envdir = File.join(Puppet[:environmentpath], environment)
+    if not Dir.exists?(envdir)
+      Dir.mkdir(envdir)
+    end
+  end
+end
+
+RSpec.configure do |config|
+
+  config.before :each do
+    @logs = []
+    Puppet::Util::Log.level = :info
+    Puppet::Util::Log.newdestination(Puppet::Test::LogCollector.new(@logs))
+
+    def test_logs
+      @logs.map(&:message)
+    end
+
+  end
+
+end

data/puppet/spec/unit/face/node/deactivate_spec.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'spec_helper'
+require 'puppet/face'
+require 'puppet/indirector/node/puppetdb'
+
+describe "node face: deactivate" do
+  let(:subject) { Puppet::Face[:node, :current] }
+
+  it "should fail if no node is given" do
+    expect { subject.deactivate }.to raise_error ArgumentError, /provide at least one node/
+  end
+
+  it "should deactivate each node using the puppetdb terminus" do
+    nodes = ['a', 'b', 'c']
+    nodes.each do |node|
+      Puppet::Node::Puppetdb.any_instance.expects(:destroy).with do |request|
+        request.key == node
+      end.returns('uuid' => "uuid_#{node}")
+    end
+
+    subject.deactivate(*nodes).should == {
+      'a' => 'uuid_a',
+      'b' => 'uuid_b',
+      'c' => 'uuid_c',
+    }
+  end
+end

data/puppet/spec/unit/face/node/status_spec.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require 'spec_helper'
+require 'puppet'
+require 'puppet/face'
+require 'puppet/network/http_pool'
+
+describe "node face: status" do
+  let(:subject) { Puppet::Face[:node, :current] }
+  let(:headers) do
+    {
+      "Accept" => "application/json",
+      "Content-Type" => "application/x-www-form-urlencoded; charset=UTF-8",
+    }
+  end
+
+  it "should fail if no node is given" do
+    expect { subject.status }.to raise_error ArgumentError, /provide at least one node/
+  end
+
+  it "should fetch the status of each node" do
+    http = stub 'http'
+    Puppet::Network::HttpPool.stubs(:http_instance).returns(http)
+
+    nodes = %w[a b c d e]
+    nodes.each do |node|
+      http.expects(:get).with("/pdb/query/v4/nodes/#{node}", headers)
+    end
+
+    subject.status(*nodes)
+  end
+
+  it "should CGI escape the node names" do
+    http = stub 'http'
+    Puppet::Network::HttpPool.stubs(:http_instance).returns(http)
+
+    node = "foo/+*&bar"
+
+    http.expects(:get).with("/pdb/query/v4/nodes/foo%2F%2B%2A%26bar", headers)
+
+    subject.status(node)
+  end
+end

data/puppet/spec/unit/face/storeconfigs_spec.rb

@@ -0,0 +1,199 @@
+#!/usr/bin/env ruby
+
+require 'puppet/util/puppetdb'
+
+if Puppet::Util::Puppetdb.puppet3compat?
+  require 'spec_helper'
+  require 'puppet/face/storeconfigs'
+  require 'json'
+  require 'puppet/util/feature'
+  require 'puppet/util/puppetdb'
+
+  describe Puppet::Face[:storeconfigs, '0.0.1'], :if => (Puppet.features.rails? && Puppet.features.sqlite?) do
+    def setup_scratch_database
+      Puppet::Rails.stubs(:database_arguments).returns(
+        :adapter => 'sqlite3',
+        :log_level => Puppet[:rails_loglevel],
+        :database => ':memory:'
+      )
+      Puppet[:railslog]     = '/dev/null'
+      Puppet::Rails.init
+    end
+
+    before :all do
+      # We have to have this block to require this file, so they get loaded on
+      # platforms where we are going to run the tests, but not on Ruby 1.8.5.
+      # Unfortunately, rspec will evaluate the describe block (but not the before
+      # block or tests) even if the conditions fail. The lack of a sqlite3 gem
+      # for Ruby 1.8.5 ensures that the condition will always be false on Ruby
+      # 1.8.5, so at this point it's safe to require this.
+      require 'puppet/indirector/catalog/active_record'
+    end
+
+    before :each do
+      setup_scratch_database
+      Puppet[:storeconfigs] = true
+      Puppet[:storeconfigs_backend] = :active_record
+    end
+
+    describe "export action" do
+      after :each do
+        FileUtils.rm_rf(@path)
+      end
+
+      before :each do
+        tempfile = Tempfile.new('export')
+        @path = tempfile.path
+        tempfile.close!
+
+        Dir.mkdir(@path)
+
+        subject.stubs(:destination_file).returns File.join(@path, 'storeconfigs-test.tar')
+      end
+
+      # Turn the filename of a gzipped tar into a hash from filename to content.
+      def tgz_to_hash(filename)
+        # List the files in the archive, ignoring directories (whose names end
+        # with /), and stripping the leading puppetdb-bak.
+        files = `tar tf #{filename}`.lines.map(&:chomp).reject { |fname| fname[-1,1] == '/'}.map {|fname| fname.sub('puppetdb-bak/', '') }
+
+        # Get the content of the files, one per line. Thank goodness they're a
+        # single line each.
+        content = `tar xf #{filename} -O`.lines.to_a
+
+        # Build a hash from filename to content. Ruby 1.8.5 doesn't like
+        # Hash[array_of_pairs], so we have to jump through hoops by flattening
+        # and splatting this list.
+        Hash[*files.zip(content).flatten]
+      end
+
+      describe "with nodes present" do
+        def notify(title, exported=false)
+          Puppet::Resource.new(:notify, title, :parameters => {:message => title}, :exported => exported)
+        end
+
+        def user(name)
+          Puppet::Resource.new(:user, name,
+                               :parameters => {:groups => ['foo', 'bar', 'baz'],
+                                               :profiles => ['stuff', 'here'] #<-- Uses an ordered list
+                                              }, :exported => true)
+        end
+
+        def save_catalog(catalog)
+          request = Puppet::Resource::Catalog.indirection.request(:save, catalog.name, catalog)
+          Puppet::Resource::Catalog::ActiveRecord.new.save(request)
+        end
+
+        before :each do
+          catalog = Puppet::Resource::Catalog.new('foo')
+
+          catalog.add_resource notify('not exported')
+          catalog.add_resource notify('exported', true)
+          catalog.add_resource user('someuser')
+          save_catalog(catalog)
+        end
+
+        it "should have the right structure" do
+          filename = subject.export
+
+          results = tgz_to_hash(filename)
+
+          results.keys.should =~ ['export-metadata.json', 'catalogs/foo.json']
+
+          metadata = JSON.load(results['export-metadata.json'])
+
+          metadata.keys.should =~ ['timestamp', 'command_versions']
+          metadata['command_versions'].should == {'replace_catalog' => 6}
+
+          catalog = JSON.load(results['catalogs/foo.json'])
+
+          catalog.keys.should =~ ['metadata', 'environment', 'certname', 'version', 'edges', 'resources', 'timestamp', 'producer_timestamp']
+
+          catalog['metadata'].should == {'api_version' => 1}
+
+          catalog['certname'].should == 'foo'
+          catalog['edges'].to_set.should == [{
+                                            'source' => {'type' => 'Stage', 'title' => 'main'},
+                                            'target' => {'type' => 'Notify', 'title' => 'exported'},
+                                            'relationship' => 'contains'},
+                                          {"source"=>{"type"=>"Stage", "title"=>"main"},
+                                           "target"=>{"type"=>"User", "title"=>"someuser"},
+                                           "relationship"=>"contains"}].to_set
+
+          catalog['resources'].should include({
+                                             'type'       => 'Stage',
+                                             'title'      => 'main',
+                                             'exported'   => false,
+                                             'tags'       => ['stage', 'main'],
+                                             'parameters' => {},
+                                           })
+
+          catalog['resources'].should include({
+                                             'type'       => 'Notify',
+                                             'title'      => 'exported',
+                                             'exported'   => true,
+                                             'tags'       => ['exported', 'notify'],
+                                             'parameters' => {
+                                               'message' => 'exported',
+                                             },
+                                           })
+
+          catalog['resources'].should include({
+                                             'type'       => 'User',
+                                             'title'      => 'someuser',
+                                             'exported'   => true,
+                                             'tags'       => ['someuser', 'user'],
+                                             'parameters' => {
+                                               'groups'   => ['foo', 'bar', 'baz'],
+                                               'profiles' => ['stuff', 'here']
+                                             },
+                                           })
+        end
+
+        it "should only include exported resources and Stage[main]" do
+          filename = subject.export
+
+          results = tgz_to_hash(filename)
+
+          results.keys.should =~ ['export-metadata.json', 'catalogs/foo.json']
+
+          catalog = JSON.load(results['catalogs/foo.json'])
+
+          catalog['certname'].should == 'foo'
+
+          catalog['edges'].map do |edge|
+            [edge['source']['type'], edge['source']['title'], edge['relationship'], edge['target']['type'], edge['target']['title']]
+          end.to_set.should == [['Stage', 'main', 'contains', 'Notify', 'exported'],
+                                ['Stage', 'main', 'contains', 'User', 'someuser']].to_set
+
+          catalog['resources'].map { |resource| [resource['type'], resource['title']] }.to_set.should == [['Notify', 'exported'], ["User", "someuser"], ['Stage', 'main']].to_set
+
+          notify = catalog['resources'].find {|resource| resource['type'] == 'Notify'}
+
+          notify['exported'].should == true
+        end
+
+        it "should exclude nodes with no exported resources" do
+          catalog = Puppet::Resource::Catalog.new('bar')
+
+          catalog.add_resource notify('also not exported')
+
+          save_catalog(catalog)
+
+          filename = subject.export
+
+          results = tgz_to_hash(filename)
+
+          results.keys.should =~ ['export-metadata.json', 'catalogs/foo.json']
+        end
+      end
+
+      it "should do nothing if there are no nodes" do
+        filename = subject.export
+
+        results = tgz_to_hash(filename)
+        results.keys.should == ['export-metadata.json']
+      end
+    end
+  end
+end