dtk-node-agent 0.5.10 → 0.5.12

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzkwMmViNTAyZjQwZTFmY2FiZTczODliMjE3YTQ0YTQ5YmU3OGIzYg==
+    NGUwMWQxZTY3ZDA1MWUyOGNkNDI3YWUxZWYxNjhkYzUwY2JlZTBkMA==
   data.tar.gz: !binary |-
-    ZGQ5NTFmZDZlZTc0MjViNTJjMzUzNDZhNzhiZGM0N2NhYjk3NGMxMQ==
+    ODc0NTA0ZDRkMzQxZmI1ODkyNTIyNDliOWRiZGM3OTRjMjcwM2Q0Yw==
 SHA512:
   metadata.gz: !binary |-
-    YzdiYjNjYTM5ZjlmODJlNjBlMzFiMzdkMzEzOGZhYTVmNTc5NzdkNDBhZmE1
-    ODcwZWQ3ZTQ1MGI0NmRlMTU4NWZlYzY0ZjlmYTg1YmJjNTAzY2I4MGQ4Y2Rl
-    ZTE1YTI5ODg5ZTAzNzgyYThlOThiNTIwMjgyMGIzNWE5ZTljMDQ=
+    MDIxZWIzNmEzYzNmNTk1N2U5ZjcyNDkyNDhjMDExNTYwMmFhNzhiODk0YmVm
+    YTkzZGRjZDJiZDg5MmVhOWE1NGU2MGQ4YjMwMGY4YjQ0YzIzMDNiMTA3MWJh
+    Yzc3NGIwYjdiMzU4M2Q4NDNjNzI5YTg4YzM0YWI2NGY0N2VlODE=
   data.tar.gz: !binary |-
-    ZjUyNTU0NmNhYjYxYzJiMDExNzExNTg3M2EzMGM3N2FjNjMxNjc2MjMxZTkz
-    NTA5ZDFkZTU1MjcyNmZiOGM2MTAxZTJiZTdhNWJlNDlmYTExZDc1NTcxZmVl
-    NDQyODdkYzczMjViYzBkMDY1OTEwYzdmZTJhNjQwMTQ3OWJiYjI=
+    OGViMTNmMTc3ODkzYmQ0YzA5YzYwODNhMjYzOTViMDA2YmRkNzZjYmI4YzZk
+    YTkxMzJjODYxNTRhYmQwYzZhMGEwNjAxMzk0NTI0NzQyYjRkODFiMTQ5NGJi
+    MzAwNjFkMTFiYjgyZjc5M2QyNTgxMmRmYzdhOGU5Njg3NDA4ODY=

data/README.md

@@ -35,6 +35,12 @@ ruby create_agent_ami.rb --region us-east-1 --ami-id ami-da0000aa --key-pair tes
 --ssh-username root --image-name dtk-agent-ubuntu-precise
 ```
 
+#### Build all supported AMI images with [packer](http://www.packer.io/) 
+```
+packer build template.json
+```  
+This will also copy images to all AWS regions.  
+
 License
 ----------------------
 DTK Node Agent is released under the GPLv3 license. Please see LICENSE for more details.

data/dtk-node-agent.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'grit', '~> 2.5.0'
   gem.add_dependency 'stomp', '~> 1.3.1'
   gem.add_dependency 'sshkeyauth', '~> 0.0.11'
-  gem.add_dependency 'serverspec'
+  gem.add_dependency 'serverspec', '~> 1.1.0'
+  gem.add_dependency 'specinfra', '~> 1.0.4'
 
 end

data/lib/dtk-node-agent/version.rb

@@ -1,3 +1,3 @@
 module DtkNodeAgent
-  VERSION="0.5.10"
+  VERSION="0.5.12"
 end

data/mcollective_additions/plugins/v2.2/agent/dev_manager.ddl

@@ -4,6 +4,6 @@ metadata    :name        => "development manager",
             :license     => "",
             :version     => "",
             :url         => "",
-            :timeout     => 2
+            :timeout     => 60
 action "inject_agent", :description => "Injects agent content and restarts mcollective" do
 end

data/mcollective_additions/plugins/v2.2/agent/dev_manager.rb

@@ -4,9 +4,8 @@ module MCollective
   module Agent
     class Dev_manager < RPC::Agent
 
-      AGENT_MCOLLECTIVE_LOCATION = "#{::MCollective::Config.instance.libdir}/mcollective/agent/"
-      @log = Log.instance
-
+      AGENT_MCOLLECTIVE_LOCATION = "#{::MCollective::Config.instance.libdir.join}/mcollective/agent/"
+ 
       action "inject_agent" do
         begin
 
@@ -31,7 +30,7 @@ module MCollective
           return ret
 
         rescue Exception => e
-          @log.error e
+          Log.instance.error e
           ret.set_status_failed!()
           error_info = { :error => { :message => "Error syncing agents: #{e}" } }
           ret.merge!(error_info)

data/mcollective_additions/plugins/v2.2/agent/execute_tests.rb

@@ -1,64 +1,104 @@
-module MCollective
-  module Agent
-    class Execute_tests < RPC::Agent
-      def initialize()
-        super()
-        @log = Log.instance
-      end
-
-      action "execute_tests" do 
-        #Get list of component modules that have spec tests
-        list_output=`ls /etc/puppet/modules/*/dtk/serverspec/spec/localhost/*/*_spec.rb`
-        regex_pattern=/modules\/(.+)\/dtk\/serverspec\/spec\/localhost\/(.+)\//
-        ModuleInfo = Struct.new(:module_name, :component_name)
-        modules_info = []
-
-        components = []
-        request[:components].each do |c|
-          if c.include? "::"
-            components << c.split("::").last
-          elsif c.include? "/"
-            components << c.split("/").last
-          else
-            components << c
-          end
-        end
-
-        list_output.each do |line|
-          match = line.match(regex_pattern)
-          components.each do |c|
-            if c.eql? match[2]
-              modules_info << ModuleInfo.new(match[1],match[2])
-            end
-          end
-        end
-
-        all_spec_results = []
-        #filter out redundant module info if any
-        modules_info = modules_info.uniq
-        modules_info.each do |module_info|
-          component_module = module_info[:module_name]
-          component = module_info[:component_name]
-
-          spec_results=`/opt/puppet-omnibus/embedded/bin/rspec /etc/puppet/modules/#{component_module}/dtk/serverspec/spec/localhost/#{component}/*_spec.rb --format j`
-          @log.info("Executing serverspec test: /etc/puppet/modules/#{component_module}/tests/serverspec/spec/localhost/#{component}/*_spec.rb")
-
-          spec_results_json = JSON.parse(spec_results)
-
-          spec_results_json['examples'].each do |spec|
-            spec_result = {}
-            spec_result.store(:module_name, component_module)
-            spec_result.store(:component_name, component)
-            spec_result.store(:test_result, spec['full_description'])
-            spec_result.store(:status, spec['status'])
-            all_spec_results << spec_result
-          end
-        end
-
-        reply[:data] = all_spec_results
-        reply[:pbuilderid] = Facts["pbuilderid"]
-        reply[:status] = :ok
-      end
-    end
-  end
-end
+module MCollective
+  module Agent
+    class Execute_tests < RPC::Agent
+      def initialize()
+        super()
+        @log = Log.instance
+      end
+
+      def pull_modules(module_version_context, git_server)
+        ENV['GIT_SHELL'] = nil #This is put in because if vcsrepo Puppet module used it sets this
+        begin
+          repo_dir = "#{ModulePath}/#{module_version_context[:implementation]}"
+          remote_repo = "#{git_server}:#{module_version_context[:repo]}"
+          opts = Hash.new
+          begin
+            if File.exists?(repo_dir)
+              @log.info("Branch already exists. Checkout to branch and pull latest changes...")
+              git_repo = ::DTK::NodeAgent::GitClient.new(repo_dir)
+              git_repo.pull_and_checkout_branch?(module_version_context[:branch],opts)
+            else
+              @log.info("Branch does not exist. Cloning branch...")
+              git_repo = ::DTK::NodeAgent::GitClient.new(repo_dir,:create=>true)
+              git_repo.clone_branch(remote_repo,module_version_context[:branch],opts)
+            end
+          rescue Exception => e
+            log_error(e)
+            #to achieve idempotent behavior; fully remove directory if any problems
+            FileUtils.rm_rf repo_dir
+            raise e
+          end
+        rescue Exception => e
+          log_error(e)
+        end
+      end
+
+      def log_error(e)
+        log_error = ([e.inspect]+backtrace_subset(e)).join("\n")
+        @log.info("\n----------------error-----\n#{log_error}\n----------------error-----")
+      end
+
+      action "execute_tests" do
+        #Get list of component modules that have spec tests
+        list_output=`ls /etc/puppet/modules/*/dtk/serverspec/spec/localhost/*/*_spec.rb`
+        regex_pattern=/modules\/(.+)\/dtk\/serverspec\/spec\/localhost\/(.+)\//
+        ModuleInfo = Struct.new(:module_name, :component_name, :full_component_name)
+        modules_info = []
+
+        components = []
+        #Strip away node part (/)...leave only part which represent full component name
+        request[:components].each do |c|
+          if c.include? "/"
+            components << c.split("/").last
+          else
+            components << c
+          end
+        end
+
+        list_output.each do |line|
+          match = line.match(regex_pattern)
+          components.each do |c|
+            if c.include? "::"
+              stripped_c = c.split("::").last
+              modules_info << ModuleInfo.new(match[1],match[2],c) if stripped_c.eql? match[2]
+            elsif c.eql? match[2]
+              modules_info << ModuleInfo.new(match[1],match[2],c)
+            end
+          end
+        end
+
+        all_spec_results = []
+        #filter out redundant module info if any
+        modules_info = modules_info.uniq
+        #Pull latest changes for modules if any
+        git_server = Facts["git-server"]
+
+        modules_info.each do |module_info|
+          component_module = module_info[:module_name]
+          component_name = module_info[:component_name]
+          full_component_name = module_info[:full_component_name]
+          #Filter out version context for modules that don't exist on node
+          filtered_version_context = request[:version_context].select { |x| x[:implementation] == module_info[:module_name] }.first
+          pull_modules(filtered_version_context,git_server)       
+
+          spec_results=`/opt/puppet-omnibus/embedded/bin/rspec /etc/puppet/modules/#{component_module}/dtk/serverspec/spec/localhost/#{component_name}/*_spec.rb --format j`
+          @log.info("Executing serverspec test: /etc/puppet/modules/#{component_module}/dtk/serverspec/spec/localhost/#{component_name}/*_spec.rb")
+
+          spec_results_json = JSON.parse(spec_results)
+          spec_results_json['examples'].each do |spec|
+            spec_result = {}
+            spec_result.store(:module_name, component_module)
+            spec_result.store(:component_name, full_component_name)
+            spec_result.store(:test_result, spec['full_description'])
+            spec_result.store(:status, spec['status'])
+            all_spec_results << spec_result
+          end
+        end
+
+        reply[:data] = all_spec_results
+        reply[:pbuilderid] = Facts["pbuilderid"]
+        reply[:status] = :ok
+      end
+    end
+  end
+end

data/mcollective_additions/plugins/v2.2/agent/ssh_agent.ddl

@@ -0,0 +1,13 @@
+metadata    :name        => "ssh agent",
+            :description => "SSH Agent allows adding of public keys, removing them and listing",
+            :author      => "Reactor8",
+            :license     => "",
+            :version     => "",
+            :url         => "",
+            :timeout     => 2
+action "grant_access", :description => "Add SSH access to host instance" do
+end
+action "revoke_access", :description => "Remove SSH access from host instance" do
+end
+action "list_access", :description => "List current SSH access for host instance" do
+end

data/mcollective_additions/plugins/v2.2/agent/ssh_agent.rb

@@ -0,0 +1,93 @@
+require 'base64'
+
+module MCollective
+  module Agent
+    class Ssh_agent < RPC::Agent
+
+      AGENT_MCOLLECTIVE_LOCATION = "#{::MCollective::Config.instance.libdir.join}/mcollective/agent/"
+      SSH_AUTH_KEYS_FILE_NAME    = "authorized_keys"
+
+      action "grant_access" do
+        validate :rsa_pub_key, String
+        validate :rsa_pub_name, String
+        validate :system_user, String
+
+        if does_user_exist?(request[:system_user])
+          begin
+            puppet_params = {
+                :name => request[:rsa_pub_name],
+                :ensure => 'present',
+                :key =>normalize_rsa_pub_key(request[:rsa_pub_key]),
+                :type => 'ssh-rsa',
+                :user => request[:system_user]
+              }
+
+            ::MCollective::Util.loadclass("MCollective::Util::PuppetRunner")
+            ::MCollective::Util::PuppetRunner.apply(:ssh_authorized_key, puppet_params)
+
+            # There is a bug where we are expiriencing issues with above changes not taking effect for no apperent reason
+            # if detected we repeat puppet apply
+
+            unless key_added?(puppet_params[:user], puppet_params[:key])
+              Log.info("Fallback, repeating SSH access grant")
+              ::MCollective::Util::PuppetRunner.apply(:ssh_authorized_key, puppet_params)
+            end
+
+            raise "We were not able to add SSH access for given node (PuppetError)" unless key_added?(puppet_params[:user], puppet_params[:key])
+
+            reply[:data] = { :message => "Access to system user '#{request[:system_user]}' has been granted for '#{request[:rsa_pub_name]}'"}
+          rescue Exception => e
+            reply[:data] = { :error => "Puppet error not able to process request, reason: '#{e.message}'" }
+          end
+        else
+          reply[:data] = { :error => "System user '#{request[:system_user]}' not found on given node" }
+        end
+        reply[:pbuilderid] = Facts["pbuilderid"]
+        reply[:status] = :ok
+      end
+
+      action "revoke_access" do
+        validate :rsa_pub_name, String
+        validate :system_user, String
+
+        if does_user_exist?(request[:system_user])
+          begin
+            ::MCollective::Util.loadclass("MCollective::Util::PuppetRunner")
+            ::MCollective::Util::PuppetRunner.apply(
+              :ssh_authorized_key,
+              {
+                :name => request[:rsa_pub_name],
+                :ensure => 'absent',
+                :type => 'ssh-rsa',
+                :user => request[:system_user]
+             }
+            )
+            reply[:data] = { :message => "Access for system user '#{request[:system_user]}' has been revoked" }
+          rescue Exception => e
+            reply[:data] = { :error => "Puppet error not able to process request, reason: '#{e.message}'" }
+          end
+        else
+          reply[:data] = { :error => "System user '#{request[:system_user]}' not found on given node" }
+        end
+
+        reply[:pbuilderid] = Facts["pbuilderid"]
+        reply[:status] = :ok
+      end
+
+      def does_user_exist?(system_user)
+        !File.open('/etc/passwd').grep(/home\/#{system_user}:/).empty?
+      end
+
+      def key_added?(system_user, pub_key)
+        results = `more /home/#{system_user}/.ssh/#{SSH_AUTH_KEYS_FILE_NAME} | grep #{pub_key}`
+        !results.empty?
+      end
+
+      def normalize_rsa_pub_key(rsa_pub_key)
+        rsa_pub_key.strip!()
+        rsa_pub_key.gsub!(/.* (.*) .*/,'\1')
+        rsa_pub_key
+      end
+    end
+  end
+end

data/mcollective_additions/plugins/v2.2/agent/sync_agent_code.rb

@@ -4,7 +4,7 @@ require 'grit'
 require 'tempfile'
 require 'fileutils'
 
-AGENT_MCOLLECTIVE_LOCATION = "#{::MCollective::Config.instance.libdir}/mcollective/agent/"
+AGENT_MCOLLECTIVE_LOCATION = "#{::MCollective::Config.instance.libdir.join}/mcollective/agent/"
 
 module MCollective
   module Agent

data/mcollective_additions/plugins/v2.2/util/puppetrunner.rb

@@ -0,0 +1,31 @@
+require 'puppet'
+
+module MCollective
+  module Util
+    class PuppetRunner
+
+      def self.apply(puppet_definition, resource_hash)
+        Puppet.settings.initialize_global_settings
+        Puppet.settings.initialize_app_defaults(Puppet::Settings.app_defaults_for_run_mode(Puppet.run_mode))
+
+        Log.info("Puppet Runner, INPUT :")
+        Log.info(puppet_definition)
+        Log.info(resource_hash.inspect)
+        Log.info("########################################################################")
+
+        pup = Puppet::Type.type(puppet_definition).new(resource_hash)
+        catalog = Puppet::Resource::Catalog.new
+        catalog.add_resource pup
+        catalog.apply()
+
+        Log.info("Puppet Runner, OUTPUT: ")
+        Log.info(Thread.current[:report_status])
+        Log.info(Thread.current[:report_info])
+        Log.info("########################################################################")
+        true
+      end
+
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dtk-node-agent
 version: !ruby/object:Gem::Version
-  version: 0.5.10
+  version: 0.5.12
 platform: ruby
 authors:
 - Rich PELAVIN
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-17 00:00:00.000000000 Z
+date: 2014-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puppet
@@ -84,16 +84,30 @@ dependencies:
   name: serverspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: specinfra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.4
 description: DTK node agent is tool used to install and configure DTK agents.
 email:
 - rich@reactor8.com
@@ -148,6 +162,8 @@ files:
 - mcollective_additions/plugins/v2.2/agent/puppet_cancel.rb
 - mcollective_additions/plugins/v2.2/agent/rpcutil.ddl
 - mcollective_additions/plugins/v2.2/agent/rpcutil.rb
+- mcollective_additions/plugins/v2.2/agent/ssh_agent.ddl
+- mcollective_additions/plugins/v2.2/agent/ssh_agent.rb
 - mcollective_additions/plugins/v2.2/agent/sync_agent_code.ddl
 - mcollective_additions/plugins/v2.2/agent/sync_agent_code.rb
 - mcollective_additions/plugins/v2.2/agent/tail.ddl
@@ -158,6 +174,7 @@ files:
 - mcollective_additions/plugins/v2.2/facts/pbuilder_facts.rb
 - mcollective_additions/plugins/v2.2/security/sshkey.ddl
 - mcollective_additions/plugins/v2.2/security/sshkey.rb
+- mcollective_additions/plugins/v2.2/util/puppetrunner.rb
 - mcollective_additions/server.cfg
 - src/etc/init.d/ec2-run-user-data
 - src/etc/logrotate.d/mcollective
@@ -182,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.5
+rubygems_version: 2.1.9
 signing_key: 
 specification_version: 4
 summary: DTK ndoe agent tool.