dtk-node-agent 0.5.10

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzkwMmViNTAyZjQwZTFmY2FiZTczODliMjE3YTQ0YTQ5YmU3OGIzYg==
+  data.tar.gz: !binary |-
+    ZGQ5NTFmZDZlZTc0MjViNTJjMzUzNDZhNzhiZGM0N2NhYjk3NGMxMQ==
+SHA512:
+  metadata.gz: !binary |-
+    YzdiYjNjYTM5ZjlmODJlNjBlMzFiMzdkMzEzOGZhYTVmNTc5NzdkNDBhZmE1
+    ODcwZWQ3ZTQ1MGI0NmRlMTU4NWZlYzY0ZjlmYTg1YmJjNTAzY2I4MGQ4Y2Rl
+    ZTE1YTI5ODg5ZTAzNzgyYThlOThiNTIwMjgyMGIzNWE5ZTljMDQ=
+  data.tar.gz: !binary |-
+    ZjUyNTU0NmNhYjYxYzJiMDExNzExNTg3M2EzMGM3N2FjNjMxNjc2MjMxZTkz
+    NTA5ZDFkZTU1MjcyNmZiOGM2MTAxZTJiZTdhNWJlNDlmYTExZDc1NTcxZmVl
+    NDQyODdkYzczMjViYzBkMDY1OTEwYzdmZTJhNjQwMTQ3OWJiYjI=

data/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+# dependcies are defined in gemspec file since to avoid duplication
+gemspec

data/Gemfile.lock

@@ -0,0 +1,18 @@
+PATH
+  remote: .
+  specs:
+    dtk-node-agent (0.1.0)
+      facter (~> 1.7.3)
+      trollop (~> 2.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    facter (1.7.3)
+    trollop (2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  dtk-node-agent!

data/README.md

@@ -0,0 +1,42 @@
+dtk-node-agent
+==============
+
+Code that is present in AMIs that server basis for nodes being spun up
+
+### Build the gem:
+`gem build dtk-node-agent.gemspec`
+
+#### Intalling the node agent on a running machine (without puppet omnibus)
+`sudo dtk-node-agent`
+
+#### Intalling the node agent on a running machine (with puppet omnibus)
+`sudo ./install_agent.sh`
+
+#### Create a new AMI image with the node agent
+```
+./create_agent_ami.rb --help
+Options:
+          --region, -r <s>:   AWS Region on which to create the AMI image
+         --aws-key, -a <s>:   AWS Access Key
+      --aws-secret, -w <s>:   AWS Secret Access Key
+  --security-group, -s <s>:   AWS Security group (default: default)
+        --key-pair, -k <s>:   AWS keypair for the new instance
+        --key-path, -e <s>:   Path to the PEM file for ssh access
+    --ssh-username, -u <s>:   SSH Username
+     --ssh-timeout, -t <i>:   Time to wait before instance is ssh ready (seconds) (default: 100)
+          --ami-id, -m <s>:   AMI id which to spin up
+      --image-name, -i <s>:   Name of the new image
+                --help, -h:   Show this message
+```
+
+example:  
+```
+ruby create_agent_ami.rb --region us-east-1 --ami-id ami-da0000aa --key-pair test_key --key-path /somepath/test_key.pem \
+--ssh-username root --image-name dtk-agent-ubuntu-precise
+```
+
+License
+----------------------
+DTK Node Agent is released under the GPLv3 license. Please see LICENSE for more details.
+
+

data/bin/dtk-node-agent

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+# Bundler and rubygems maintain a set of directories from which to
+# load gems. If Bundler is loaded, let it determine what can be
+# loaded. If it's not loaded, then use rubygems. But do this before
+# loading any dtk-node-agent code, so that our gem loading system is sane.
+if not defined? ::Bundler
+  begin
+    require 'rubygems'
+  rescue LoadError
+  end
+end
+
+require 'dtk-node-agent/installer'
+
+DTK::NodeAgent::Installer.run(ARGV)

data/dtk-node-agent.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/dtk-node-agent/version', __FILE__)
+prod_version_path = File.expand_path('../lib/dtk-node-agent/prod_version', __FILE__)
+if File.exist?("#{prod_version_path}.rb")
+  require prod_version_path
+else
+  DtkNodeAgent::PROD_VERSION = nil
+end
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Rich PELAVIN"]
+  gem.email         = ["rich@reactor8.com"]
+  gem.description   = %q{DTK node agent is tool used to install and configure DTK agents.}
+  gem.summary       = %q{DTK ndoe agent tool.}
+  gem.homepage      = "https://github.com/rich-reactor8/dtk-node-agent"
+  gem.licenses      = ["GPL-3.0"]
+
+  gem.files = %w(README.md Gemfile Gemfile.lock dtk-node-agent.gemspec)
+  gem.files += Dir.glob("bin/**/*")
+  gem.files += Dir.glob("lib/**/*")
+  gem.files += Dir.glob("puppet_additions/**/*")
+  gem.files += Dir.glob("mcollective_additions/**/*")
+  gem.files += Dir.glob("src/**/*")
+  
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "dtk-node-agent"
+  gem.require_paths = ["lib"]
+  gem.version       = DtkNodeAgent::PROD_VERSION || "#{DtkNodeAgent::VERSION}.#{ARGV[3]}".chomp(".")
+
+  gem.add_dependency 'puppet', '~> 3.3.2'
+  gem.add_dependency 'facter', '~> 1.7.3'
+  gem.add_dependency 'grit', '~> 2.5.0'
+  gem.add_dependency 'stomp', '~> 1.3.1'
+  gem.add_dependency 'sshkeyauth', '~> 0.0.11'
+  gem.add_dependency 'serverspec'
+
+end

data/lib/config/install.config

@@ -0,0 +1,12 @@
+{
+	:puppet_version => '2.7.23',
+	:mcollective_version => 2.2,
+	:puppetlabs_el5_rpm_repo => 'http://yum.puppetlabs.com/el/5/products/i386/puppetlabs-release-5-7.noarch.rpm',
+	:puppetlabs_el6_rpm_repo => 'http://yum.puppetlabs.com/el/6/products/i386/puppetlabs-release-6-7.noarch.rpm',
+	:rpm_forge_el5_X86_64_repo => 'http://apt.sw.be/redhat/el5/en/x86_64/rpmforge/RPMS/rpmforge-release-0.5.3-1.el5.rf.x86_64.rpm',
+	:rpm_forge_el5_i686_repo => 'http://apt.sw.be/redhat/el5/en/i386/rpmforge/RPMS/rpmforge-release-0.5.3-1.el5.rf.i386.rpm',
+	:rpm_forge_el6_X86_64_repo => 'http://apt.sw.be/redhat/el6/en/x86_64/rpmforge/RPMS/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm',
+	:rpm_forge_el6_i686_repo => 'http://apt.sw.be/redhat/el6/en/i386/rpmforge/RPMS/rpmforge-release-0.5.3-1.el6.rf.i686.rpm',
+	:puppet_omnibus_deb64 => 'http://dtk-storage.s3.amazonaws.com/puppet-omnibus_2.7.23-fpm0_amd64.deb',
+	:puppet_omnibus_rpm64 => 'http://dtk-storage.s3.amazonaws.com/puppet-omnibus-2.7.23.fpm0-1.x86_64.rpm'
+}

data/lib/dtk-node-agent/installer.rb

@@ -0,0 +1,142 @@
+module DTK
+  module NodeAgent
+    class Installer
+
+      # read configuration
+      CONFIG = eval(File.open(File.expand_path('../config/install.config', File.dirname(__FILE__))) {|f| f.read })
+
+      def self.run(argv)
+        require 'optparse'
+        require 'facter'
+        require 'fileutils'
+        require 'dtk-node-agent/version'
+
+        @@options = parse(argv)
+
+        unless Process.uid == 0
+          puts "dtk-node-agent must be started with root/sudo privileges."
+          exit(1)
+        end
+
+        if Facter.operatingsystem == 'Debian' || Facter.operatingsystem == 'Ubuntu'
+              # set up apt and install packages
+              shell "apt-get update --fix-missing"
+              shell "apt-get install -y build-essential wget curl git"
+              shell "wget http://apt.puppetlabs.com/puppetlabs-release-#{Facter.lsbdistcodename}.deb"
+              puts "Installing Puppet Labs repository..."
+              shell "dpkg -i puppetlabs-release-#{Facter.lsbdistcodename}.deb"
+              shell "apt-get update"
+              shell "rm puppetlabs-release-#{Facter.lsbdistcodename}.deb"
+              # install mcollective
+              puts "Installing MCollective..."
+              shell "apt-get -y install mcollective"
+            elsif Facter.operatingsystem == 'CentOS' || Facter.operatingsystem == 'RedHat'
+              shell "yum -y install yum-utils wget"
+              case Facter.operatingsystemmajrelease
+              when "5"
+                shell "rpm -ivh #{CONFIG[:puppetlabs_el5_rpm_repo]}"
+                Facter.architecture == 'X86_64' ? (shell "rpm -ivh #{CONFIG[:rpm_forge_el5_X86_64_repo]}") : (shell "rpm -ivh #{CONFIG[:rpm_forge_el5_i686_repo]}")
+              when "6"
+                shell "rpm -ivh #{CONFIG[:puppetlabs_el6_rpm_repo]}"
+                Facter.architecture == 'X86_64' ? (shell "rpm -ivh #{CONFIG[:rpm_forge_el6_X86_64_repo]}") : (shell "rpm -ivh #{CONFIG[:rpm_forge_el6_i686_repo]}")
+                shell "yum-config-manager --disable rpmforge-release"
+                shell "yum-config-manager --enable rpmforge-extras"
+              else
+                puts "#{Facter.operatingsystem} #{Facter.operatingsystemmajrelease} is not supported. Exiting now..."
+                exit(1)
+              end
+              puts "Installing MCollective..."
+              shell "yum -y install mcollective"
+              shell "chkconfig mcollective on"
+              shell "yum -y install git"
+              # install ec2-run-user-data init script
+              FileUtils.cp("#{base_dir}/src/etc/init.d/ec2-run-user-data", "/etc/init.d/ec2-run-user-data") unless File.exist?("/etc/init.d/ec2-run-user-data")
+              shell "chmod +x /etc/init.d/ec2-run-user-data"
+              shell "chkconfig --level 345 ec2-run-user-data on"
+
+            else
+              echo "Unsuported OS for automatic agent installation. Exiting now..."
+              exit(1)
+            end
+
+            puts "Installing additions for MCollective and Puppet..."
+            install_additions
+
+          end       
+
+
+          private
+          
+          def self.parse(argv)
+            options = {}
+            parser = OptionParser.new do |opts|
+              opts.banner = <<-BANNER
+              usage:
+              
+              dtk-node-agent [-p|--puppet-version] [-v|--version]
+              BANNER
+              opts.on("-d",
+                "--debug",
+                "enable debug mode")  { |v| options[:debug] = true }      
+              opts.on_tail("-v",
+                "--version",
+                "Print the version and exit.") do
+                puts ::DtkNodeAgent::VERSION
+                exit(0)
+              end
+              opts.on_tail("-h",
+                "--help",
+                "Print this help message.") do
+                puts parser
+                exit(0)
+              end
+            end
+
+            parser.parse!(argv)
+
+            options
+            
+          rescue OptionParser::InvalidOption => e
+            $stderr.puts e.message
+            exit(12)
+          end
+
+          def self.shell(cmd)
+            puts "running: #{cmd}" if @@options[:debug]
+            output = `#{cmd}`
+            puts output if @@options[:debug]
+            if $?.exitstatus != 0
+              puts "Executing command \`#{cmd}\` failed"
+              puts "Command output:"
+              puts output
+            end
+          end
+
+          def self.install_additions
+        # create puppet group
+        shell "groupadd puppet" unless `grep puppet /etc/group`.include? "puppet"
+        # create necessary dirs
+        [   '/var/log/puppet/',
+          '/var/lib/puppet/lib/puppet/indirector',
+          '/etc/puppet/modules',
+          '/usr/share/mcollective/plugins/mcollective'
+          ].map! { |p| FileUtils.mkdir_p(p) unless File.directory?(p) }
+        # copy puppet libs
+        FileUtils.cp_r(Dir.glob("#{base_dir}/puppet_additions/puppet_lib_base/puppet/indirector/*"), "/var/lib/puppet/lib/puppet/indirector/")
+        # copy r8 puppet module
+        FileUtils.cp_r(Dir.glob("#{base_dir}/puppet_additions/modules/r8"), "/etc/puppet/modules")
+        # copy mcollective plugins
+        FileUtils.cp_r(Dir.glob("/usr/libexec/mcollective/mcollective/*"), "/usr/share/mcollective/plugins/mcollective") if File.directory?("/usr/libexec/mcollective/")        
+        FileUtils.cp_r(Dir.glob("#{base_dir}/mcollective_additions/plugins/v#{CONFIG[:mcollective_version]}/*"), "/usr/share/mcollective/plugins/mcollective")
+
+        # copy mcollective config
+        FileUtils.cp_r("#{base_dir}/mcollective_additions/server.cfg", "/etc/mcollective", :remove_destination => true)
+      end
+
+      def self.base_dir
+        File.expand_path('../..', File.dirname(__FILE__))
+      end
+
+    end
+  end
+end

data/lib/dtk-node-agent/version.rb

@@ -0,0 +1,3 @@
+module DtkNodeAgent
+  VERSION="0.5.10"
+end

data/mcollective_additions/plugins/README.md

@@ -0,0 +1 @@
+Deprecate all mcolelctive agents in the os specfic directories

data/mcollective_additions/plugins/v1.2/agent/discovery.rb

@@ -0,0 +1,39 @@
+module MCollective
+  module Agent
+    # Discovery agent for The Marionette Collective
+    #
+    # Released under the Apache License, Version 2
+    class Discovery
+      attr_reader :timeout, :meta
+
+      def initialize
+        config = Config.instance.pluginconf
+
+        @timeout = 5
+        @timeout = config["discovery.timeout"].to_i if config.include?("discovery.timeout")
+
+        @meta = {:license => "Apache License, Version 2",
+                 :author => "R.I.Pienaar <rip@devco.net>",
+                 :timeout => @timeout,
+                 :name => "Discovery Agent",
+                 :version => MCollective.version,
+                 :url => "http://www.marionette-collective.org",
+                 :description => "MCollective Discovery Agent"}
+      end
+
+      def handlemsg(msg, stomp)
+        reply = "unknown request"
+
+        case msg[:body]
+          when "ping"
+            reply = "pong"
+
+          else
+            reply = "Unknown Request: #{msg[:body]}"
+        end
+
+        reply
+      end
+    end
+  end
+end

data/mcollective_additions/plugins/v1.2/agent/get_log_fragment.ddl

@@ -0,0 +1,15 @@
+metadata    :name        => "get log fragment",
+                        :description => "get log fragment",
+                        :author      => "Reactor8",
+                        :license     => "",
+                        :version     => "",
+                        :url         => "",
+                        :timeout     => 20
+action "get", :description => "get log data fragment" do
+  display :always
+  %w{status error data pbuilderid}.each do |k|
+    output k.to_sym,
+      :description => k.capitalize,
+      :display_as => k.capitalize
+  end		
+end

data/mcollective_additions/plugins/v1.2/agent/get_log_fragment.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+
+module MCollective
+  #TODO: shoudl go in common area
+  LogFileHandles = {:task_id => Hash.new, :top_task_id => Hash.new}
+
+  module Agent
+    class Get_log_fragment < RPC::Agent
+            metadata    :name        => "get log fragment",
+                        :description => "get log fragment",
+                        :author      => "Reactor8",
+                        :license     => "",
+                        :version     => "",
+                        :url         => "",
+                        :timeout     => 20
+      def initialize()
+        super()
+        @log = Log.instance
+      end
+      def get_action()
+        validate :key, String
+        validate :value, String
+        log_file_dir = "/var/log/puppet" 
+        key = request[:key]
+        value = request[:value]
+        lines = get_log_fragment(log_file_dir,key,value)
+        pbuilderid = Facts["pbuilderid"]
+        if lines.nil?
+          error_msg = "Cannot find log fragment matching #{key}=#{value}"
+          error_response = {
+            :status => :failed, 
+            :error => {
+              :message => error_msg
+            },
+            :pbuilderid => pbuilderid
+          }
+          @log.error(error_msg)
+          reply.data = error_response
+        else
+          ok_response = {
+            :status => :ok,
+            :data => lines,
+            :pbuilderid => pbuilderid
+          }
+          reply.data = ok_response
+        end
+      end
+     private
+      def get_log_fragment(log_file_dir,key,value)
+        #flush file if it is open
+        if file_handle = (LogFileHandles[key.to_sym]||{})[value]
+          file_handle.flush
+        end
+ 
+        delim = key.to_sym == :top_task_id ? "\\." : ":"
+        string_key = "#{key}:#{value.to_s}#{delim}"
+        ret = Array.new
+        matches = Dir["#{log_file_dir}/*.log"].grep(Regexp.new(string_key))
+        if matches.size == 0
+          return nil
+        end
+        matching_file = matches.size == 1 ?
+          matches.first :
+          #this finds teh most recent file
+          matches.map{|file|[file,File.mtime(file)]}.sort{|a,b|b[1]<=>a[1]}.first[0]
+        begin
+          f = File.open(matching_file)
+          until f.eof
+            ret << f.readline.chop
+          end
+        ensure
+          f.close
+        end
+        ret
+      end
+    end
+  end
+end