right_chimp 1.0

data/lib/right_chimp/exec/ExecSSH.rb

@@ -0,0 +1,45 @@
+#
+# Run an SSH script
+#
+module Chimp
+  class ExecSSH < Executor
+    attr_accessor :ssh_user
+    
+    def initialize(h={})
+      super(h)
+      @ssh_user = h[:ssh_user]
+    end
+    
+    def run
+      host = @server['ip_address'] || @server['ip-address'] || nil
+      @ssh_user ||= "root"
+      
+      if host == nil
+        @server.settings
+        host = @server['ip_address'] || @server['ip-address']
+      end
+      
+      run_with_retry do
+        Log.debug "ssh #{@ssh_user}@#{host} \"#{@exec}\""
+        success = system("ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no #{@ssh_user}@#{host} \"#{@exec}\"")
+        
+        if not $?.success?
+          raise "SSH failed with status: #{$?}"
+        end
+      end
+    end
+    
+    def describe_work
+      return "ExecSSH job_id=#{@job_id} command=\"#{@exec}\" server=\"#{@server['nickname']}\""
+    end
+    
+    def info
+      return @exec.to_s
+    end
+    
+    def target
+      return @server['nickname']
+    end
+    
+  end
+end

data/lib/right_chimp/exec/Executor.rb

@@ -0,0 +1,180 @@
+#
+# Superclass for Executors-- objects that run things on servers
+#
+
+module Chimp
+  class Executor
+    attr_accessor :server, :array, :exec, :inputs, :template, :owner, :group,
+                  :job_id, :status, :dry_run, :verbose, :quiet, :timeout,
+                  :retry_count, :retry_sleep, :time_start, :time_end
+    
+    attr_reader   :error, :results
+    
+    STATUS_NONE =     :none
+    STATUS_RUNNING =  :running
+    STATUS_RETRYING = :retrying
+    STATUS_ERROR =    :error
+    STATUS_DONE =     :done
+
+    def initialize(h={})
+      @server = h[:server]            || nil
+      @array = h[:array]              || nil
+      @template = h[:template]        || nil
+
+      @job_id = h[:job_id]            || nil
+      @group = h[:group]              || nil
+      @exec = h[:exec]                || nil
+      @inputs = h[:inputs]            || nil
+
+      @verbose = h[:verbose]          || false
+      
+      @retry_count = h[:retry_count].to_i || 0
+      @retry_sleep = h[:retry_sleep].to_i || 30
+      @timeout = h[:timeout].to_i         || 3600
+      
+      @error = nil
+      @status = STATUS_NONE
+      @owner = nil
+      @dry_run = false
+      @quiet = false
+      
+      @time_start = nil
+      @time_end = nil
+      @results = nil
+    end
+
+    #
+    # Return total execution time (real) of a job
+    #
+    def get_total_exec_time
+      if @time_start == nil
+        return 0
+      elsif @time_end == nil
+        return Time.now.to_i - @time_start.to_i
+      else
+        return @time_end.to_i- @time_start.to_i
+      end
+    end
+    
+    #
+    # Convenience method to requeue
+    #
+    def requeue
+      @group.requeue(self.job_id)
+    end
+    
+    #
+    # Convenience method to cancel
+    #
+    def cancel
+      @group.cancel(self.job_id)
+    end
+
+    def run
+      raise "run method must be overridden"
+    end
+      
+    #
+    # return info on what this executor does -- eg name of script or command
+    #
+    def info
+      raise "unimplemented"
+    end
+    
+    def target
+      return "UNKNOWN"
+    end
+
+        
+    protected
+    
+    #
+    # Run a unit of work with retries
+    # This is called from the subclass with a code block to yield to
+    #
+    def run_with_retry(&block)
+      @status = STATUS_RUNNING
+      @time_start = Time.now
+      Log.info self.describe_work_start unless @quiet
+      
+      #
+      # The inner level of exception handling here tries to catch anything
+      # that can be easily retired or failed-- normal exceptions.
+      #
+      # The outer level of exception handling handles weird stuff; for example,
+      # sometimes rest_connection raises RuntimeError exceptions... 
+      #
+      # This fixes acu75562.
+      #
+      begin
+        begin
+          yield if not @dry_run
+        
+          if @owner != nil
+            @status = STATUS_DONE
+            @group.job_completed
+          else
+            Log.warn "Ownership of job_id #{job_id} lost. User cancelled operation?"
+          end
+        
+        rescue SystemExit, Interrupt => ex
+          $stderr.puts "Exiting!"
+          raise ex
+        
+        rescue Interrupt => ex
+          name = @array['name'] if @array
+          name = @server['name'] || @server['nickname'] if @server
+          Log.error self.describe_work_error
+        
+          if @retry_count > 0
+            @status = STATUS_RETRYING
+            Log.error "Error executing on \"#{name}\". Retrying in #{@retry_sleep} seconds..."
+            @retry_count -= 1
+            sleep @retry_sleep
+            retry
+          end
+        
+          @status = STATUS_ERROR
+          @error = ex
+          Log.error "Error executing on \"#{name}\": #{ex}"
+          
+        ensure 
+          @time_end = Time.now
+          Log.info self.describe_work_done unless @quiet
+        end
+        
+      rescue RuntimeError => ex
+        Log.error "Caught RuntimeError: #{ex}. Aborting job."
+        Log.error ex.inspect
+        Log.error ex.backtrace
+        @status = STATUS_ERROR
+        @error = ex
+      end
+    end
+    
+    #
+    # This method should be overridden on Executor subclasses
+    # to provide a human readable description of the work
+    # being performed.
+    #
+    def describe_work
+      return "#{self.class.name} job_id=#{@job_id}"
+    end
+    
+    def describe_work_start
+      return("#{self.describe_work} status=START")
+    end
+
+    def describe_work_done
+      return("#{self.describe_work} status=END time=#{@time_end.to_i-@time_start.to_i}s")
+    end
+    
+    def describe_work_done_long
+      return("#{self.describe_work} status=END time_start=#{@time_start.to_i} time_end=#{@time_end.to_i} time_total=#{@time_end.to_i-@time_start.to_i}")
+    end
+    
+    def describe_work_error
+      return("#{self.describe_work} status=ERROR")
+    end    
+  end
+end

data/lib/right_chimp/queue/ChimpQueue.rb

@@ -0,0 +1,187 @@
+module Chimp
+  #
+  # The ChimpQueue is a singleton that contains the
+  # chimp work queue
+  #
+  class ChimpQueue
+    include Singleton
+    
+    attr_accessor :delay, :retry_count, :max_threads, :group
+    
+    def initialize
+      @delay = 0
+      @retry_count = 0
+      @max_threads = 10
+      @workers_never_exit = true
+      @threads = []
+      
+      self.reset!
+    end
+    
+    #
+    # Reset the queue and the :default group
+    #
+    # This doesn't do anything to the groups's jobs
+    #
+    def reset!
+      @group = {}
+      @group[:default] = ParallelExecutionGroup.new(:default)
+    end
+    
+    #
+    # Start up queue runners
+    #
+    def start
+      self.sort_queues!
+      
+      for i in (1..max_threads)
+        @threads << Thread.new(i) do
+          worker = QueueWorker.new
+          worker.delay = @delay
+          worker.retry_count = @retry_count
+          worker.run
+        end
+      end
+    end
+    
+    #
+    # Push a task into the queue
+    #
+    def push(g, w)
+      raise "no group specified" unless g
+      create_group(g) if not ChimpQueue[g]
+      ChimpQueue[g].push(w)
+    end
+    
+    def create_group(name, type = :parallel, concurrency = 1)
+      type = :parallel
+      Log.debug "Creating new execution group #{name} type=#{type} concurrency=#{concurrency}"
+      new_group = ExecutionGroupFactory.from_type(type)
+      new_group.group_id = name
+      new_group.concurrency = concurrency
+      ChimpQueue[name] = new_group
+    end
+    
+    #
+    # Grab the oldest work item available
+    #
+    def shift
+      r = nil
+      @group.values.each do |group|
+        if group.ready?
+          r = group.shift
+          break
+        end
+      end      
+      return(r)
+    end
+    
+    #
+    # Wait until a group is done
+    #
+    def wait_until_done(g, &block)
+      while @group[g].running? 
+        @threads.each do |t|
+          t.join(1)
+          yield          
+        end
+      end
+    end
+    
+    #
+    # Quit - empty the queue and wait for remaining jobs to complete
+    # 
+    def quit
+      i = 0
+      @group.keys.each do |group|
+        wait_until_done(group) do
+          if i < 30
+            sleep 1
+            i += 1
+            print "."
+          else
+            break
+          end
+        end
+      end
+      
+      @threads.each { |t| t.kill }
+      puts " done."
+    end
+    
+    #
+    # Run all threads forever (used by chimpd)
+    #
+    def run_threads
+      @threads.each do |t| 
+        t.join(5)
+      end
+    end
+    
+    #
+    # return the total number of queued (non-executing) objects
+    #
+    def size
+      s = 0
+      @group.values.each do |group|
+        s += group.size
+      end
+      return(s)
+    end
+    
+    #
+    # Allow the groups to be accessed as ChimpQueue.group[:foo]
+    #
+    def self.[](group)
+      return ChimpQueue.instance.group[group]
+    end
+    
+    def self.[]=(k,v)
+      ChimpQueue.instance.group[k] = v
+    end
+    
+    #
+    # Return an array of all jobs with the requested
+    # status.
+    #
+    def get_jobs_by_status(status)
+      r = []
+      @group.values.each do |group| 
+        v = group.get_jobs_by_status(status)
+        if v != nil and v != []
+          r += v
+        end
+      end
+      
+      return r
+    end
+    
+    def get_job(id)
+      jobs = self.get_jobs
+      
+      jobs.each do |j|
+        return j if j.job_id == id
+      end
+    end
+    
+    def get_jobs
+      r = []
+      @group.values.each do |group|
+        group.get_jobs.each { |job| r << job }
+      end
+      
+      return r
+    end
+    
+    ############################################################# 
+    protected
+    
+    #
+    # Sort all the things, er, queues
+    #
+    def sort_queues!
+      return @group.values.each { |group| group.sort! }
+    end
+    
+  end
+end

data/lib/right_chimp/queue/ExecutionGroup.rb

@@ -0,0 +1,277 @@
+module Chimp
+  
+  #
+  # Factory
+  #
+  class ExecutionGroupFactory
+    def self.from_type(type)
+      if type == :serial
+        return SerialExecutionGroup.new(nil)
+      elsif type == :parallel
+        return ParallelExecutionGroup.new(nil)
+      else
+        raise "invalid execution group type specified"
+      end
+    end
+  end
+  
+  #
+  # An ExecutionGroup contains a set of Executors to be processed
+  #
+  # Only the subclasses SerialExecutionGroup and ParallelExecutionGroup
+  # should be used directly.
+  #
+  class ExecutionGroup
+    attr_accessor :group_id, :description, :concurrency
+    attr_reader   :time_start, :time_end
+    
+    def initialize(new_group_id=nil)
+      @group_id = new_group_id
+      @queue = []
+      @jobs_by_id = {}
+      @log = nil
+      @time_start = nil
+      @time_end = nil
+      @concurrency = 1
+    end
+        
+    #
+    # Add something to the work queue
+    #  
+    def push(j)
+      raise "invalid work" if j == nil 
+      j.job_id = IDManager.get if j.job_id == nil
+      j.group = self
+      @queue.push(j)
+      @jobs_by_id[j.job_id] = j
+    end
+    
+    #
+    # Take something from the queue
+    #
+    def shift
+      x = @queue.shift
+      @time_start = Time.now if @time_start == nil
+      return x
+    end
+    
+    #
+    # Return a hash of the results
+    #
+    def results
+      return self.get_jobs.map do |task|
+        next if task == nil
+        next if task.server == nil
+        
+        {
+          :job_id => task.job_id,
+          :name   => task.info,
+          :host   => task.server['nickname'] || task.server['name'],
+          :status => task.status,
+          :error  => task.error,
+          :total  => self.get_total_execution_time(task.status, task.time_start, task.time_end),
+          :start  => task.time_start,
+          :end    => task.time_end,
+          :worker => task
+        }
+      end
+    end
+    
+    #
+    # Size of the active queue
+    #
+    def size
+      return @queue.size
+    end
+    
+    #
+    # Sort queue by server nickname
+    #
+    def sort!
+      if @queue != nil
+        @queue.sort! do |a,b|
+          a.server['nickname'] <=> b.server['nickname']
+        end
+      end
+    end
+    
+    #
+    # Reset the queue
+    #
+    def reset!
+      @queue = []
+    end
+    
+    #
+    # Get all jobs
+    #
+    def get_jobs
+      @jobs_by_id.values
+    end
+    
+    #
+    # Get all job ids
+    #
+    def get_job_ids
+      @jobs_by_id.keys
+    end
+    
+    #
+    # Get a particular job
+    #
+    def get_job(i)
+      @jobs_by_id[i]
+    end
+    
+    #
+    # Get jobs by status
+    #
+    def get_jobs_by_status(status)
+      r = []
+      @jobs_by_id.values.each do |i|
+        r << i if i.status == status.to_sym || status.to_sym == :all
+      end
+      return r
+    end
+    
+    def job_completed
+      @time_end = Time.now
+    end
+    
+    #
+    # Reset all jobs and bulk set them
+    #
+    def set_jobs(jobs=[])
+      self.reset!
+      jobs.each do |job|
+        self.push(job)
+      end
+    end
+    
+    #
+    # An execution group is "ready" if it has work that can be done;
+    # see implementation in child classes.
+    #
+    def ready?
+      raise "unimplemented"
+    end
+    
+    #
+    # An execution group is "done" if nothing is queued or running
+    #
+    def done?
+      return (get_jobs_by_status(Executor::STATUS_NONE).size == 0 && get_jobs_by_status(Executor::STATUS_RUNNING).size == 0)
+    end
+    
+    #
+    # Is this execution group running anything?
+    #
+    def running?
+      total_jobs_running = get_jobs_by_status(Executor::STATUS_NONE).size + 
+          get_jobs_by_status(Executor::STATUS_RUNNING).size +
+          get_jobs_by_status(Executor::STATUS_RETRYING).size
+      return(total_jobs_running > 0)
+    end
+
+    #
+    # Requeue all failed jobs
+    #
+    def requeue_failed_jobs!
+      get_jobs_by_status(Executor::STATUS_ERROR).each do |job|
+        requeue(job.job_id)
+      end
+    end
+
+    #
+    # Requeue a job by id
+    #
+    def requeue(id)
+      job = @jobs_by_id[id]
+      job.status = Executor::STATUS_NONE
+      job.owner = nil
+      job.time_start = Time.now
+      job.time_end = nil
+      self.push(job)
+    end
+    
+    #
+    # Cancel a job by id
+    #
+    def cancel(id)
+      Log.warn "Cancelling job id #{id}"
+      job = @jobs_by_id[id]
+      job.status = Executor::STATUS_ERROR
+      job.owner = nil
+      job.time_end = Time.now
+      @queue.delete(job)
+    end
+    
+    #
+    # Return total execution time
+    #
+    def get_total_exec_time
+      if @time_start == nil
+        return 0
+      elsif @time_end == nil
+        return Time.now.to_i - @time_start.to_i
+      else
+        return @time_end.to_i- @time_start.to_i
+      end
+    end
+    
+    #
+    # Print out ExecutionGroup information
+    #
+    def to_s
+      return "#{self.class}[#{group_id}]: ready=#{self.ready?} total_jobs=#{@jobs_by_id.size} queued_jobs=#{self.size}"
+    end
+    
+    ###################################
+    protected
+    ###################################
+    
+    #
+    # Return total execution time or -1 for errors
+    #
+    def get_total_execution_time(status, time_begin, time_end)
+      return(status != :error ? time_end.to_i - time_begin.to_i : -1)
+    end
+
+  end
+  
+  #
+  # SerialExecutionGroup: run only one job at a time
+  #
+  class SerialExecutionGroup < ExecutionGroup
+    def ready?
+      return get_jobs_by_status(Executor::STATUS_RUNNING).size == 0 && get_jobs_by_status(Executor::STATUS_NONE).size > 0
+    end
+    
+    def short_name
+      "S"
+    end
+  end
+  
+  #
+  # ParallelExecutionGroup: run multiple jobs at once
+  #
+  class ParallelExecutionGroup < ExecutionGroup
+    def initialize(new_group_id)
+      super(new_group_id)
+      @concurrency = 25
+    end
+    
+    #
+    # FIXME - we're not currently using the @concurrency setting to limit execution
+    #         due to an unknown bug...
+    #
+    def ready?
+      return (get_jobs_by_status(Executor::STATUS_NONE).size > 0) # and get_jobs_by_status(Executor::STATUS_RUNNING).size < @concurrency)
+    end
+    
+    def short_name
+      "P"
+    end
+  end
+  
+end

data/lib/right_chimp/queue/QueueWorker.rb

@@ -0,0 +1,42 @@
+#
+# QueueWorker objects take work from the Queue and process it
+# Each QueueWorker runs in its own thread... nothing fancy going on here
+#
+module Chimp
+  class QueueWorker
+    attr_accessor :delay, :retry_count, :never_exit
+    
+    def initialize
+      @delay = 0
+      @retry_count = 0
+      @never_exit = true
+    end
+    
+    #
+    # Grab work items from the ChimpQueue and process them
+    # Only stop is @ever_exit is false
+    #
+    def run
+      while @never_exit
+        work_item = ChimpQueue.instance.shift()
+
+        begin
+          if work_item != nil
+            work_item.retry_count = @retry_count
+            work_item.owner = Thread.current.object_id
+            work_item.run
+            sleep @delay
+          else
+            sleep 1
+          end
+        
+        rescue StandardError => ex
+          $stderr.puts "Exception in QueueWorker.run: #{ex}"
+          puts ex.inspect
+          puts ex.backtrace
+        end
+      end
+    end
+    
+  end
+end