right_chimp 1.0

data/lib/right_chimp/IDManager.rb

@@ -0,0 +1,18 @@
+#
+# Utility class for generating sequential job ids
+#
+module Chimp
+  class IDManager
+    @@id = 0
+    @@mutex = Mutex.new
+    
+    def self.get
+      r = nil
+      @@mutex.synchronize do
+        @@id = @@id + 1
+        r = @@id
+      end
+      return r
+    end
+  end
+end

data/lib/right_chimp/Log.rb

@@ -0,0 +1,34 @@
+module Chimp
+  class Log
+    @@logger = Logger.new(STDOUT)
+    @@logger.sev_threshold=Logger::INFO
+    
+    def self.logger=(l)
+      @@logger = l
+    end
+
+    def self.threshold=(level)
+      @@logger.sev_threshold=level
+    end
+
+    def self.default_level(level)
+      @@logger.level = level
+    end
+
+    def self.debug(m)
+      @@logger.debug(m)
+    end
+    
+    def self.info(m)
+      @@logger.info(m)
+    end
+
+    def self.warn(m)
+      @@logger.warn(m)
+    end
+    
+    def self.error(m)
+      @@logger.error(m)
+    end    
+  end
+end

data/lib/right_chimp/daemon/ChimpDaemon.rb

@@ -0,0 +1,416 @@
+#
+# ChimpDaemon.rb
+#
+# Classes for the Chimp Daemon (chimpd)
+#
+
+module Chimp
+  class ChimpDaemon
+    attr_accessor :verbose, :debug, :port, :concurrency, :delay, :retry_count, :dry_run, :logfile
+    attr_reader :queue, :running
+    
+    include Singleton
+    
+    def initialize
+      @verbose     = false
+      @debug       = false
+      @port        = 9055
+      @concurrency = 50
+      @delay       = 0
+      @retry_count = 0      
+      @threads     = []
+      @running     = false
+      @queue       = ChimpQueue.instance
+    end
+    
+    #
+    # Main entry point for chimpd command line application
+    #
+    def run
+      install_signal_handlers
+      parse_command_line
+      
+      puts "chimpd #{VERSION} launching with #{@concurrency} workers"
+      spawn_queue_runner
+      spawn_webserver
+      run_forever
+    end
+    
+    #
+    # Parse chimpd command line options
+    #
+    def parse_command_line
+      begin
+        opts = GetoptLong.new(
+          [ '--logfile', '-l',      GetoptLong::REQUIRED_ARGUMENT ],
+          [ '--verbose', '-v',      GetoptLong::NO_ARGUMENT ],
+          [ '--quiet',   '-q',      GetoptLong::NO_ARGUMENT ],
+          [ '--concurrency', '-c',  GetoptLong::REQUIRED_ARGUMENT ],
+          [ '--delay', '-d',        GetoptLong::REQUIRED_ARGUMENT ],
+          [ '--retry', '-y',        GetoptLong::REQUIRED_ARGUMENT ],
+          [ '--port', '-p',         GetoptLong::REQUIRED_ARGUMENT ],
+          [ '--exit', '-x', 				GetoptLong::NO_ARGUMENT ]
+        )
+        
+        opts.each do |opt, arg|
+          case opt
+            when '--logfile', '-l'
+              @logfile = arg
+              Log.logger = Logger.new(@logfile)
+            when '--concurrency', '-c'
+              @concurrency = arg.to_i
+            when '--delay', '-d'
+              @delay = arg.to_i
+            when '--retry', '-y'
+              @retry_count = arg.to_i
+            when '--verbose', '-v'
+              @verbose = true
+            when '--quiet',   '-q'
+              @quiet = true
+            when '--port', '-p'
+              @port = arg
+            when '--exit', '-x'
+            	uri = "http://localhost:9055/admin"
+							response = RestClient.post uri, { 'shutdown' => true }.to_yaml
+							exit 0
+          end
+        end
+      rescue GetoptLong::InvalidOption => ex
+        puts "Syntax: chimpd [--logfile=<name>] [--concurrency=<c>] [--delay=<d>] [--retry=<r>] [--port=<p>] [--verbose]"
+        exit 1
+      end
+      
+      #
+      # Set up logging/verbosity
+      #
+      Chimp.set_verbose(@verbose, @quiet)
+      
+      if not @verbose
+      	ENV['REST_CONNECTION_LOG'] = "/dev/null"
+      	ENV['RESTCLIENT_LOG'] = "/dev/null"
+      end
+    end
+    
+    #
+    # Spawn the ChimpQueue threads
+    #
+    def spawn_queue_runner      
+      @queue.max_threads = @concurrency
+      @queue.delay = @delay
+      @queue.retry_count = @retry_count
+      @queue.start
+      @running = true
+    end
+    
+    #
+    # Spawn a WEBrick Web server
+    #
+    def spawn_webserver
+      opts = {
+        :BindAddress  => "localhost", 
+        :Port         => @port,
+        :MaxClients   => 50,
+        :RequestTimeout => 60
+      }
+      
+      if not @verbose
+        opts[:Logger] = WEBrick::Log.new("/dev/null")
+        opts[:AccessLog] = [nil, nil]
+      end
+
+      @server = ::WEBrick::HTTPServer.new(opts)      
+      @server.mount('/',         DisplayServlet)
+      @server.mount('/display',  DisplayServlet)
+      @server.mount('/job',      JobServlet)
+      @server.mount('/group',    GroupServlet)
+      @server.mount('/admin',    AdminServlet)
+      
+      #
+      # WEBrick threads
+      #
+      @threads << Thread.new(1001) do
+        @server.start
+      end
+    end
+    
+    #
+    # Process requests forever until we're killed
+    #
+    def run_forever
+      @running = true
+      while @running
+        @threads.each do |t|
+          t.join(5)
+        end
+      end
+    end
+    
+    #
+    # Trap signals to exit cleanly
+    #
+    def install_signal_handlers
+      ['INT', 'TERM'].each do |signal| 
+        trap(signal) do
+          self.quit
+        end
+      end
+    end
+    
+    #
+    # Quit by waiting for all chimp jobs to finish, not allowing
+    # new jobs on the queue, and killing the web server.
+    #
+    # TODO: call @queue.quit, but with a short timeout?
+    #
+    def quit
+      @running = false
+      @server.shutdown
+      sleep 5
+      exit 0
+    end
+    
+    #
+    # GenericServlet -- servlet superclass
+    #
+    class GenericServlet < WEBrick::HTTPServlet::AbstractServlet
+      def get_verb(req)
+        r = req.request_uri.path.split('/')[2]
+      end
+      
+      def get_id(req)
+        uri_parts = req.request_uri.path.split('/')
+        id = uri_parts[-2]
+        return id
+      end
+      
+      #
+      # Get the body of the request-- assume YAML
+      #
+      def get_payload(req)
+        begin
+          return YAML::load(req.body)
+        rescue StandardError => ex
+          return nil
+        end
+      end
+    end # GenericServlet
+  
+    #
+    # AdminServlet - admin functions
+    #
+    class AdminServlet < GenericServlet
+      def do_POST(req, resp)
+        payload = self.get_payload(req)
+        shutdown = payload['shutdown'] || false
+
+        if shutdown == true
+        	ChimpDaemon.instance.quit
+        end
+        
+        raise WEBrick::HTTPStatus::OK		
+      end
+    end # AdminServlet  
+    
+    #
+    # GroupServlet - group information and control
+    #
+    # http://localhost:9055/group/default/running
+    #
+    class GroupServlet < GenericServlet
+      #
+      # GET a group by name and status
+      # /group/<name>/<status>
+      #
+      def do_GET(req, resp)
+        jobs = {}
+       
+        group_name = req.request_uri.path.split('/')[-2]
+        filter     = req.request_uri.path.split('/')[-1]
+        
+        g = ChimpQueue[group_name.to_sym]
+        raise WEBrick::HTTPStatus::NotFound, "Group not found" unless g
+        jobs = g.get_jobs_by_status(filter)
+        resp.body = jobs.to_yaml
+        raise WEBrick::HTTPStatus::OK
+      end
+      
+      #
+      # POST to a group to trigger a group action
+      # /group/<name>/<action>
+      #
+      def do_POST(req, resp)
+        group_name = req.request_uri.path.split('/')[-2]
+        filter     = req.request_uri.path.split('/')[-1]
+        payload    = self.get_payload(req)
+
+        if filter == 'create'
+          ChimpQueue.instance.create_group(group_name, payload['type'], payload['concurrency'])
+      
+        elsif filter == 'retry'
+          group = ChimpQueue[group_name.to_sym]
+          raise WEBrick::HTTPStatus::NotFound, "Group not found" unless group
+        
+          group.requeue_failed_jobs!
+          raise WEBrick::HTTPStatus::OK
+          
+        else
+          raise WEBrick::HTTPStatus::PreconditionFailed.new("invalid action")
+        end
+      end
+      
+    end
+    
+    #
+    # JobServlet - job control
+    #
+    # HTTP body is a yaml serialized chimp object
+    #
+    class JobServlet < GenericServlet
+      def do_POST(req, resp)
+        id      = -1
+        job_id  = self.get_id(req)
+        verb    = self.get_verb(req)
+
+        payload = self.get_payload(req)
+        raise WEBrick::HTTPStatus::PreconditionFailed.new('missing payload') unless payload
+        
+        q = ChimpQueue.instance
+        group = payload.group
+        
+        #
+        # Ask chimpd to process a Chimp object directly
+        #
+        if verb == 'process' or verb == 'add'
+          payload.interactive = false
+          tasks = payload.process          
+          tasks.each do |task|
+            q.push(group, task)
+          end
+          
+        elsif verb == 'update'
+          puts "UPDATE"
+          q.get_job(job_id).status = payload.status
+        end
+
+        resp.body = {
+          'id' => id
+        }.to_yaml
+        
+        raise WEBrick::HTTPStatus::OK
+      end
+        
+      def do_GET(req, resp)
+        id          = self.get_id(req)
+        verb        = self.get_verb(req)
+        job_results = "OK"
+        queue       = ChimpQueue.instance
+        
+        #
+        # check for special job ids
+        #
+        jobs = []        
+        jobs << queue.get_job(id.to_i)
+        
+        jobs = queue.get_jobs_by_status(:running) if id == 'running'
+        jobs = queue.get_jobs_by_status(:error)   if id == 'error'
+        jobs = queue.get_jobs                     if id == 'all'
+        
+        raise WEBrick::HTTPStatus::PreconditionFailed.new('invalid or missing job_id #{id}') unless jobs.size > 0
+        
+        #
+        # ACK a job -- mark it as successful even if it failed
+        #
+        if req.request_uri.path =~ /ack$/
+          jobs.each do |j|
+            j.status = Executor::STATUS_DONE
+          end
+          
+          resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
+        
+        #
+        # retry a job
+        #
+        elsif req.request_uri.path =~ /retry$/
+          jobs.each do |j|
+            j.requeue
+          end
+          
+          resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
+        
+        #
+        # cancel an active job
+        #
+        elsif req.request_uri.path =~ /cancel$/
+          jobs.each do |j|
+          	j.cancel if j.respond_to? :cancel
+          end
+          
+        	resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
+        
+        #
+        # produce a report
+        #
+        elsif req.request_uri.path =~ /report$/
+          results = ["group_name,type,job_id,script,target,start_time,end_time,total_time,status"]
+          jobs.each do |j|
+            results << [j.group.group_id, j.class.to_s.sub("Chimp::",""), j.job_id, j.info, j.target, j.time_start, j.time_end, j.get_total_exec_time, j.status].join(",")
+          end
+          
+          queue.group.values.each do |g|
+            results << [g.group_id, g.class.to_s.sub("Chimp::",""), "", "", "", g.time_start, g.time_end, g.get_total_exec_time, ""].join(",")
+          end
+          
+          job_results = results.join("\n") + "\n"
+          
+          resp['Content-type'] = "text/csv"
+          resp['Content-disposition'] = "attachment;filename=chimp.csv"
+        end
+        
+        #
+        # return a list of the results
+        #
+        resp.body = job_results
+        raise WEBrick::HTTPStatus::OK
+      end
+    end # JobServlet
+    
+    #
+    # DisplayServlet
+    #
+    class DisplayServlet < GenericServlet
+      def do_GET(req, resp)
+        job_filter = self.get_verb(req) || "running"
+
+        if not @template
+          if ENV['CHIMP_TEST'] != 'TRUE'
+            template_file_name = File.join(Gem.dir, 'gems', 'chimp-' + VERSION, 'lib/chimp/templates/all_jobs.erb')
+          else
+            template_file_name = 'lib/chimp/templates/all_jobs.erb'
+          end
+          
+          @template = ERB.new(File.read(template_file_name), nil, ">")
+        end
+        
+        queue = ChimpQueue.instance        
+        jobs = queue.get_jobs
+        group_name = nil
+                
+        if job_filter == "group"
+          group_name = req.request_uri.path.split('/')[-1]
+          g = ChimpQueue[group_name.to_sym]
+          jobs = g.get_jobs if g
+        end
+
+        count_jobs_running = queue.get_jobs_by_status(:running).size
+        count_jobs_queued  = queue.get_jobs_by_status(:none).size
+        count_jobs_failed  = queue.get_jobs_by_status(:error).size
+        count_jobs_done    = queue.get_jobs_by_status(:done).size
+        
+        resp.body = @template.result(binding)
+        raise WEBrick::HTTPStatus::OK
+      end
+    end # DisplayServlet
+  end
+end
+
+

data/lib/right_chimp/daemon/ChimpDaemonClient.rb

@@ -0,0 +1,74 @@
+module Chimp
+  #
+  # The ChimpDaemonClient contains the code for communicating with the 
+  # RESTful chimpd Web service
+  #
+  class ChimpDaemonClient
+    #
+    # Submit a Chimp object to a remote chimpd
+    #
+    def self.submit(host, port, chimp_object)
+      uri = "http://#{host}:#{port}/job/process"
+      response = RestClient.post uri, chimp_object.to_yaml
+      
+      if response.code > 199 and response.code < 300
+        begin
+          id = YAML::load(response.body)['id']
+        rescue StandardError => ex
+          puts ex
+        end
+      else
+        $stderr.puts "error submitting to chimpd! response code: #{reponse.code}"
+        return false
+      end
+      
+      puts "chimpd submission complete"
+      return true
+    end
+    
+    #
+    # quit a remote chimpd
+    #
+    def self.quit(host, port)
+      response = RestClient.post "http://#{host}:#{port}/admin", { 'shutdown' => 'true' }.to_yaml
+      return response.code
+    end
+
+    #
+    # retrieve job info from a remote chimpd
+    #
+    def self.retrieve_job_info(host, port)
+      uri = "http://#{host}:#{port}/job/0"
+      response = RestClient.get uri
+      jobs = YAML::load(response.body)
+      return jobs
+    end
+    
+    def self.retrieve_group_info(host, port, group_name, status)
+      uri = "http://#{host}:#{port}/group/#{group_name}/#{status}"
+      response = RestClient.get uri
+      group = YAML::load(response.body)
+      return group
+    end
+    
+    def self.set_job_status(host, port, job_id, status)
+      uri = "http://#{host}:#{port}/job/#{job_id}/update"
+      response = RestClient.post uri, { :status => status}
+      return YAML::load(response.body)
+    end
+    
+    def self.create_group(host, port, name, type, concurrency)
+      uri = "http://#{host}:#{port}/group/#{name}/create"
+      payload = { 'type' => type, 'concurrency' => concurrency }.to_yaml
+      response = RestClient.post(uri, payload)
+      return YAML::load(response.body)
+    end
+    
+    def self.retry_group(host, port, group_name)
+      uri = "http://#{host}:#{port}/group/#{group_name}/retry"
+      response = RestClient.post(uri, {})
+      return YAML::load(response.body)
+    end
+   
+  end
+end

data/lib/right_chimp/exec/ExecArray.rb

@@ -0,0 +1,43 @@
+#
+# Run a RightScript on an array
+#
+module Chimp
+  class ExecArray < Executor
+    def run
+      run_with_retry do
+        audit_entry = []
+        
+        #
+        # If we have API 0.1 access, then use the 0.1 version of this call.
+        # Otherwise use the 1.0 version.
+        #
+        if @array.internal
+          audit_entry = @array.internal.run_script_on_instances(@exec, @server['href'], {})
+        else
+          audit_entry = @array.run_script_on_instances(@exec, @server['href'], {})
+        end
+        
+        if audit_entry
+          audit_entry.each do |a|
+            a.wait_for_completed
+          end
+        else
+          Log.warn "No audit entries returned for job_id=#{@job_id}"
+        end
+      end
+    end
+    
+    def describe_work
+      return "ExecArray job_id=#{@job_id} script=\"#{@exec['right_script']['name']}\" server=\"#{@server['nickname']}\""
+    end
+    
+    def info
+      return @exec['right_script']['name']
+    end
+    
+    def target
+      return @server['nickname']
+    end
+    
+  end
+end

data/lib/right_chimp/exec/ExecCallback.rb

@@ -0,0 +1,15 @@
+#
+# Run an SSH script
+#
+module Chimp
+  class ExecCallback < Executor
+    def run
+      response = RestClient.get @uri
+      if response.code > 199 and response.code < 300
+        return true
+      else
+        return false
+      end
+    end
+  end
+end

data/lib/right_chimp/exec/ExecNoop.rb

@@ -0,0 +1,9 @@
+module Chimp
+  class ExecNoop < Executor
+    def run
+      run_with_retry do
+        # do nothing
+      end
+    end
+  end
+end

data/lib/right_chimp/exec/ExecReport.rb

@@ -0,0 +1,57 @@
+#
+# Report
+#
+# :fields is a comma seperated list of fields to report on,
+# and are taken from the RestConnection Server objects directly.
+#
+# Example: ip-address,name,href,private-ip-address,aws-id,ec2-instance-type,dns-name,locked
+#
+module Chimp
+  class ExecReport < Executor
+    attr_reader :server, :fields
+    attr_writer :server, :fields
+    
+    def info
+      return "report on server #{fields.inspect}"
+    end
+
+    def run
+      run_with_retry do
+        output = []
+        
+        #
+        # The API and rest_connection return very different data
+        # depending on the API call made (here, tag query vs. array)
+        # so the logic to load the server information and tags is
+        # messy.
+        #
+        begin
+          s = Server.find(@server.href)
+          s.settings
+          response = ::Tag.search_by_href(s.current_instance_href)
+        rescue Exception => ex
+          s = @server
+          response = ::Tag.search_by_href(s.href)
+        end
+
+        s.tags = [] unless s.tags
+        response.each do |t|
+          s.tags += [ t['name'] ]
+        end
+        
+        @fields.split(",").each do |f|
+          if f =~ /^tag=([^,]+)/
+            tag_search_string = $1
+            s.tags.each do |tag|
+              output << tag if tag =~ /^#{tag_search_string}/
+            end
+          else
+            output << s.params[f]
+          end
+        end
+        
+        puts output.join(",")
+      end
+    end
+  end
+end

data/lib/right_chimp/exec/ExecRightScript.rb

@@ -0,0 +1,35 @@
+#
+# Run a RightScript
+#
+module Chimp
+  class ExecRightScript < Executor
+  
+    def run
+      options = {:ignore_lock => true}.merge(@inputs)
+      
+      if @timeout < 300
+        Log.error "timeout was less than 5 minutes! resetting to 5 minutes"
+        @timeout = 300
+      end
+    
+      run_with_retry do
+        audit_entry = server.run_executable(@exec, options)
+        audit_entry.wait_for_state("completed", @timeout)
+        @results = audit_entry.summary
+      end
+    end
+    
+    def describe_work
+      return "ExecRightScript job_id=#{@job_id} script=\"#{@exec['right_script']['name']}\" server=\"#{@server['nickname']}\""
+    end
+    
+    def info
+      return @exec['right_script']['name']
+    end
+    
+    def target
+      return @server['nickname']
+    end
+    
+  end
+end