skynet 0.9.1 → 0.9.2

data/lib/skynet/skynet_launcher.rb

@@ -1,7 +1,11 @@
+# FIXME: should be a module
 class Skynet
   include SkynetDebugger
   def self.new(options={})            
-    if options[:worker_type] or ARGV.detect {|a| a =~ /worker_type/ }
+    if ARGV.detect {|a| a == 'console' }
+      ARGV.delete('console')
+      Skynet::Console.start
+    elsif options[:worker_type] or ARGV.detect {|a| a =~ /worker_type/ }
       Skynet::Worker.start(options)
     else
       Skynet::Manager.start(options)

data/lib/skynet/skynet_manager.rb

@@ -7,6 +7,10 @@ class Skynet
   end
 
   class Manager
+    
+    class Error < StandardError
+    end
+    
     include SkynetDebugger
 
     Skynet::CONFIG[:PERCENTAGE_OF_TASK_ONLY_WORKERS]    ||= 0.7
@@ -16,27 +20,31 @@ class Skynet
       "MANAGER"
     end                  
         
-
-    attr_accessor :required_libs
+    attr_accessor :required_libs, :queue_id
+    attr_reader   :config
     
-    def initialize(script_path,workers_requested,adlibs=[])
-      info "Skynet Launcher Path: [#{@script_path}]"
-      @script_path          = script_path
-      @mutex                = Mutex.new
-      @workers_requested    = workers_requested
-      @required_libs        = adlibs
+    def initialize(options)      
+      raise Error.new("You must provide a script path to Skynet::Manager.new.") unless options[:script_path]
+      @script_path          = options[:script_path]
+      # info "Skynet Launcher Path: [#{@script_path}]"
+      @workers_requested    = options[:workers]  || 4
+      @required_libs        = options[:adlibs]   || []
+      @queue_id             = options[:queue_id] || 0
       @number_of_workers    = 0
       @workers_by_type      = {:master => [], :task => [], :any => []}
       @signaled_workers     = []
       @workers_running      = {}
+      @workers_restarting   = 0
       @all_workers_started  = false                      
+      @config               = Skynet::Config.new      
+      @mutex                = Mutex.new
     end   
     
     def start_workers
       setup_signals
       
       starting = workers_to_start(@workers_requested)
-      warn "Starting #{starting} workers.  #{@workers_requested - starting} already running."
+      warn "Starting #{starting} workers.  QUEUE: #{config.queue_name_by_id(queue_id)} #{@workers_requested - starting} already running."
       add_worker(starting)
     end
     
@@ -61,7 +69,7 @@ class Skynet
     def check_started_workers
       workers = []                              
       begin
-        50.times do |ii|
+        100.times do |ii|
           workers = worker_queue
           warn "Checking started workers, #{workers.size} out of #{@number_of_workers} after the #{(ii+1)}th try..."
           break if workers.size >= @number_of_workers        
@@ -97,20 +105,23 @@ class Skynet
     end
     
     def check_workers
-      q_pids = worker_queue_pids || []
+      worker_queue = self.worker_queue
+      q_pids = worker_queue_pids(worker_queue) || []          
       info "Checking on #{@number_of_workers} workers..." unless @shutdown
-      check_running_pids(q_pids)
-      check_number_of_workers(q_pids)
+      check_running_pids(worker_queue)
+      check_number_of_workers(worker_queue)
       true          
     end
 
-    def check_running_pids(q_pids)
+    def check_running_pids(worker_queue)
+      # There are workers running that are not in the queue.  When does this happen?
+      q_pids = worker_queue_pids(worker_queue) || []
       if @workers_running.keys.size > q_pids.size
          (@workers_running.keys - q_pids).each do |wpid|
            error "Missing worker #{wpid} from worker queue. Removing and/or killing."
            Process.kill("TERM",wpid) if worker_alive?(wpid)
            @workers_running.delete(wpid)
-           q_pids.delete(wpid)
+           q_pids.delete(wpid)          
          end
       end
       
@@ -125,13 +136,45 @@ class Skynet
       end
       q_pids
     end                          
+
+    def check_number_of_workers(worker_queue=self.worker_queue)
+      q_pids = worker_queue_pids(worker_queue) || []
+      if @shutdown         
+        worker_shutdown(worker_queue)
+        if q_pids.size < 1
+          exit
+        end        
+      elsif @workers_restarting > 0
+        if @workers_requested - q_pids.size != 0
+          restarting = @workers_requested - q_pids.size  
+          warn "RESTART MODE: Expected #{@number_of_workers} workers.  #{q_pids.size} running. #{restarting} are still restarting"          
+        else                    
+          warn "RESTART MODE: Expected #{@number_of_workers} workers.  #{q_pids.size} running."
+        end
+        @workers_restarting = @workers_requested - q_pids.size
+        
+      elsif q_pids.size != @number_of_workers
+        starting = 0        
+        if q_pids.size.to_f / @workers_requested.to_f < 0.85
+          starting = @workers_requested - q_pids.size  
+          error "Expected #{@number_of_workers} workers.  #{q_pids.size} running. Starting #{starting}"          
+          @number_of_workers = q_pids.size
+          add_worker(starting)          
+        else          
+          
+          error "Expected #{@number_of_workers} workers.  #{q_pids.size} running."
+          @number_of_workers = q_pids.size
+        end
+      end
+    end
                         
-    def worker_shutdown(q_pids)
+    def worker_shutdown(worker_queue)
+      q_pids = worker_queue_pids(worker_queue) || []
       if not @masters_dead
-        warn "Shutting down masters.  #{q_pids.size} workers still running." if q_pids.size > 0
         workers_to_kill = worker_queue.select do |w| 
           w.map_or_reduce == "master" and @workers_running.include?(w.process_id)
         end                           
+        warn "Shutting down masters.  #{q_pids.size} workers still running." if q_pids.size > 0
 
         worker_pids_to_kill = workers_to_kill.collect { |w| w.process_id }
         if worker_pids_to_kill and not worker_pids_to_kill.empty?
@@ -143,10 +186,10 @@ class Skynet
           signal_workers("INT")
           @masters_dead = true
           sleep 1
-          return check_number_of_workers(worker_queue_pids)
+          return check_number_of_workers()
         else
           sleep 4
-          return check_number_of_workers(worker_queue_pids)
+          return check_number_of_workers()
         end
       else
         warn "Shutting down.  #{q_pids.size} workers still running." if q_pids.size > 0
@@ -155,27 +198,7 @@ class Skynet
         info "No more workers running."
       end        
     end      
-    
-    def check_number_of_workers(q_pids)
-      if @shutdown         
-        worker_shutdown(q_pids)
-        if q_pids.size < 1
-          exit
-        end        
-      elsif q_pids.size != @number_of_workers
-        if q_pids.size.to_f / @workers_requested.to_f < 0.85
-          starting = @workers_requested - q_pids.size  
-          error "Expected #{@number_of_workers} workers.  #{q_pids.size} running. Starting #{starting}"          
-          @number_of_workers += starting 
-          add_worker(starting)          
-        else          
-          error "Expected #{@number_of_workers} workers.  #{q_pids.size} running."
-          @number_of_workers = q_pids.size
-        end
-      end
-
-    end
-    
+        
     def take_worker_status(worker_process_id)
       begin
         mq.take_worker_status({
@@ -222,11 +245,12 @@ class Skynet
           worker_types[:any] += 1
         end                          
         cmd = "#{@script_path} --worker_type=#{worker_type}"
+        cmd << " --queue_id=#{queue_id}"
         cmd << " -r #{required_libs.join(' -r ')}" if required_libs and not required_libs.empty?
         wpid = self.fork_and_exec(cmd)
         @workers_by_type[worker_type] ||= []
         @workers_by_type[worker_type] << wpid
-        warn "Adding Worker ##{ii} PID: #{wpid} WORKER_TYPE?:#{worker_type}"
+        warn "Adding Worker ##{ii} PID: #{wpid} QUEUE: #{queue_id}, WORKER_TYPE?:#{worker_type}"
         @mutex.synchronize do
           @number_of_workers += 1
         end
@@ -252,7 +276,6 @@ class Skynet
         @number_of_workers -= 1
         @workers_running.delete(wpid)      
         warn "REMOVING WORKER #{wpid}"
-        # error "SHUTTING DOWN #{wpid} MR:",worker_queue.detect{|w|w.process_id == wpid}
         @signaled_workers << wpid
         Process.kill("INT",wpid)      
       end                       
@@ -290,6 +313,19 @@ class Skynet
       check_started_workers
     end
 
+# ===========================
+# = XXX THIS IS A HORRIBLE HACK =
+# ===========================
+    def restart_worker(wpid)
+      info "RESTARTING WORKER #{wpid}"
+      @mutex.synchronize do
+        @workers_running.delete(wpid)
+        @workers_restarting += 1
+      end
+      Process.kill("HUP",wpid)
+      sleep Skynet::CONFIG[:WORKER_CHECK_DELAY]
+    end
+
     def restart_workers
       @all_workers_started = false
       signal_workers("HUP")
@@ -350,7 +386,7 @@ class Skynet
       mq.read_all_worker_statuses(hostname)
     end
     
-    def worker_queue_pids
+    def worker_queue_pids(worker_queue=self.worker_queue)
       worker_queue.collect {|w| w.process_id}
     end        
 
@@ -375,8 +411,19 @@ class Skynet
       options[:remove_workers] ||= nil
       options[:use_rails]      ||= false
       options[:required_libs]  ||= []
+      options[:workers]        ||= Skynet::CONFIG[:NUMBER_OF_WORKERS] || 4
+      options[:pid_file]       ||= Skynet::CONFIG[:SKYNET_PIDS_FILE]
+      options[:script_path]    ||= Skynet::CONFIG[:LAUNCHER_PATH]
+
+      config = Skynet::Config.new
+
       OptionParser.new do |opt|
-        opt.banner = "Usage: skynet [options]"
+        opt.banner = %{Usage: 
+        > skynet [options]
+
+        You can also run:
+        > skynet console [options]
+        }
         opt.on('', '--restart-all-workers', 'Restart All Workers') do |v| 
           puts "Restarting ALL workers on ALL machines."
           begin
@@ -416,13 +463,23 @@ class Skynet
         opt.on('-r', '--required LIBRARY', 'Require the specified libraries') do |v|
           options[:required_libs] << File.expand_path(v)
         end
-
+        opt.on('-q', '--queue QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v| 
+          options[:queue] = v
+        end               
+        opt.on('-i', '--queue_id queue_id', 'Which queue should these workers use (default 0).') do |v| 
+          options[:queue_id] = v.to_i
+        end               
         opt.parse!(ARGV)
       end
+      if options[:queue]
+        if options[:queue_id]
+          raise Skynet::Error.new("You may either provide a queue_id or a queue, but not both.")
+        end
+        options[:queue_id] = config.queue_id_by_name(options[:queue])
+      else
+        options[:queue_id] ||= 0        
+      end
       
-      options[:workers]   ||=  Skynet::CONFIG[:NUMBER_OF_WORKERS] || 4
-      options[:pid_file]  ||=  File.dirname(Skynet::CONFIG[:SKYNET_PIDS_FILE]) + "/skynet_worker.pid"
-
       options[:required_libs].each do |adlib|
         begin
           require adlib
@@ -469,7 +526,7 @@ class Skynet
 
         begin                                                                                                                   
           info "STARTING THE MANAGER!!!!!!!!!!!"
-          @manager = Skynet::Manager.new(Skynet::CONFIG[:LAUNCHER_PATH],options[:workers],options[:required_libs])
+          @manager = Skynet::Manager.new(options)
           DRb.start_service(Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL], @manager)
           info "WORKER MANAGER URI: #{DRb.uri}"
           @manager.start_workers

data/lib/skynet/skynet_message.rb

@@ -3,43 +3,59 @@ class Skynet
 
     include SkynetDebugger
     
-    class BadMessage < Skynet::Error
-    end
+    class BadMessage < Skynet::Error; end
     
     class << self
       attr_accessor :fields
     end
 
-    self.fields = {
-      0  => :tasktype,
-      1  => :drburi,
-      2  => :task_id,
-      3  => :job_id,
-      4  => :payload,
-      5  => :payload_type,
-      6  => :name,
-      7  => :expiry,
-      8  => :expire_time,
-      9  => :iteration,
-      10 => :version
-    }
+    self.fields = [
+     :tasktype,
+     :drburi,
+     :task_id,
+     :job_id,
+     :payload,
+     :payload_type,
+     :name,
+     :expiry,
+     :expire_time,
+     :iteration,
+     :version,
+     :retry,
+     :queue_id
+    ]
 
-    self.fields.values.each do |method| 
-      next if method == :payload
-      next if method == :tasktype
-      next if method == :payload_type
+    self.fields.each do |method| 
+      next if [:payload, :tasktype, :payload_type].include?(method)
       attr_accessor method
     end
     
     attr_reader :payload_type, :tasktype
+    
+    def self.new_task_message(task,job)
+      self.new(
+        :job_id       => job.job_id,
+        :expire_time  => job.start_after,
+        :version      => job.version,
+        :queue_id     => job.queue_id || 0,
+        :iteration    => 0,
+        :tasktype     => :task, 
+        :task_id      => task.task_id,
+        :payload      => task,
+        :payload_type => task.task_or_master,
+        :expiry       => task.result_timeout, 
+        :name         => task.name,       
+        :retry        => task.retry
+      )
+    end
   
     def initialize(opts)
       if opts.is_a?(Array)
-        self.class.fields.each do |ii, field|
+        self.class.fields.each_with_index do |field, ii|
           self.send("#{field}=",opts[ii] || nil)
         end
       elsif opts
-        self.class.fields.values.each do |field|
+        self.class.fields.each do |field|
           value = opts[field] || opts[field.to_s] || nil
           self.send("#{field}=",value) if value
         end
@@ -47,6 +63,7 @@ class Skynet
         if opts_raw_payload
           self.raw_payload = opts_raw_payload
         end
+        self.retry ||= 0
       end
       self.payload      
     end
@@ -95,15 +112,15 @@ class Skynet
     end
     
     def to_a
-      self.class.fields.keys.sort.collect do |ii|
-        self.send(self.class.fields[ii])
+      self.class.fields.collect do |field|
+        self.send(field)
       end
     end
     
     def to_hash
       hash = {}
-      self.class.fields.keys.sort.collect do |ii|
-        hash[self.class.fields[ii]] = self.send(self.class.fields[ii])
+      self.class.fields.each do |field|
+        hash[field] = self.send(field)
       end                                          
       hash
     end   
@@ -120,85 +137,66 @@ class Skynet
       expire_time * 2
     end
 
-    ####### TEMPLATES ############
-    def self.next_task_template(version=nil,payload_type=nil)
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :expire_time
-          (0 .. Time.now.to_i)
-        when :tasktype
-          :task
-        when :version
-          version
-        when :payload_type
-          payload_type
-        when :iteration
-          (0..6)
-        else
-          nil
-        end
+    def self.next_task_template(version=nil, payload_type=nil, queue_id=0)
+      template = {
+        :expire_time  => (0 .. Time.now.to_i),
+        :tasktype     => :task,
+        :queue_id     => queue_id,
+        :version      => version,
+        :payload_type => payload_type,
+        :iteration    => (0..Skynet::CONFIG[:MAX_RETRIES]),
+      }
+
+      fields.collect do |field|
+        template[field]
       end
     end
   
     def self.result_template(job_id,tasktype=:result)
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :tasktype
-          tasktype
-        when :job_id
-          job_id
-        else
-          nil
-        end
-      end      
+      template = {
+        :tasktype => tasktype,
+        :job_id   => job_id
+      }
+      fields.collect do |field|
+        template[field]
+      end
     end
   
     def self.result_message(message,result,tasktype=:result, resulttype=:result)
-      message_array = fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field      
-        when :tasktype
-          tasktype
-        when :payload
-          result
-        when :payload_type
-          resulttype
-        else
-          message.send(fields[ii])
-        end
+      template = {
+        :tasktype     => tasktype,
+        :payload      => result,
+        :payload_type => resulttype
+      }
+
+      fields.each do |field|
+        template[field] = message.send(field) unless template.has_key?(field)
       end
-      new(message_array)
+      new(template)
     end
   
     def result_message(result,tasktype=:result, resulttype=:result)
       self.class.result_message(self,result,tasktype,resulttype)
     end
   
-    def self.outstanding_tasks_template(iteration=nil)
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :tasktype
-          :task
-        when :iteration
-          iteration
-        else
-          nil
-        end
+    def self.outstanding_tasks_template(iteration=nil,queue_id=0)
+      template = {
+        :tasktype  => :task,
+        :queue_id  => queue_id,
+        :iteration => iteration
+      }
+      fields.collect do |field|
+        template[field]
       end
     end  
 
-    def self.outstanding_results_template
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :tasktype
-          :result
-        else
-          nil
-        end
+    def self.outstanding_results_template(queue_id=0)
+      template = {
+        :tasktype => :result,
+        :queue_id => queue_id
+      }
+      fields.collect do |field|
+        template[field]
       end
     end
   
@@ -211,16 +209,15 @@ class Skynet
     end
   
     def self.error_template(message)
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :tasktype
-          message.tasktype
-        when :drburi, :version, :task_id
-          message.send(fields[ii])
-        else
-          nil
-        end
+      template = {
+        :tasktype  => message.tasktype,
+        :drburi    => message.drburi, 
+        :version   => message.version, 
+        :task_id   => message.task_id, 
+        :queue_id  => message.queue_id
+      }
+      fields.collect do |field|
+        template[field]
       end
     end
     
@@ -229,19 +226,29 @@ class Skynet
     end  
 
     def self.fallback_task_message(message)
-       opts = Hash.new
-       fields.values.each do |field|
-         case field
-         when :iteration
-           opts[:iteration]   = message.iteration + 1
-         when :expire_time
-           opts[:expire_time] = Time.now.to_i + message.expiry
+       template = {}
+       if message.retry
+         if (message.retry and message.iteration >= message.retry)
+           template[:iteration] = -1
          else
-           opts[field] = message.send(field)
+           template[:iteration] = message.iteration + 1
          end
+       # Originally I was gonna do this for map and reduce, but we don't know that here, just whether its a master.
+       elsif message.payload_type.to_sym == :master and Skynet::CONFIG[:DEFAULT_MASTER_RETRY] and message.iteration >= Skynet::CONFIG[:DEFAULT_MASTER_RETRY]
+         template[:iteration] = -1           
+       elsif Skynet::CONFIG[:MAX_RETRIES] and message.iteration >= Skynet::CONFIG[:MAX_RETRIES]
+         template[:iteration] = -1           
+       else
+         template[:iteration] = message.iteration + 1
        end
-       # debug "BUILDING NEXT FALLBACK TASK MESSAGE OFF"#, opts
-       Skynet::Message.new(opts)
+
+       template[:expire_time] = Time.now.to_i + message.expiry
+
+       fields.each do |field|
+         template[field] = message.send(field) unless template.has_key?(field)
+       end
+       # debug "BUILDING NEXT FALLBACK TASK MESSAGE OFF"#, template
+       Skynet::Message.new(template)
     end      
   
     def fallback_task_message
@@ -249,18 +256,16 @@ class Skynet
     end
 
     def self.fallback_template(message)
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :tasktype
-          message.tasktype
-        when :drburi, :version, :task_id
-          message.send(field)
-        when :iteration
-          (1..20)
-        else
-          nil
-        end
+      template = {
+        :tasktype  => message.tasktype,
+        :drburi    => message.drburi, 
+        :version   => message.version, 
+        :task_id   => message.task_id, 
+        :queue_id  => message.queue_id,        
+        :iteration => (1..Skynet::CONFIG[:MAX_RETRIES]),        
+      }
+      fields.collect do |field|
+        template[field]
       end
     end
   
@@ -285,49 +290,43 @@ class Skynet
     end
 
     def self.template
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :tasktype
-          :current_worker_rev
-        else
-          nil
-        end
+      template = {
+        :tasktype  => :current_worker_rev
+      }
+      fields.collect do |field|
+        template[field]
       end
     end
     
     def template
-      fields.keys.sort.collect do |ii|
-        field = fields[ii]
-        case field
-        when :tasktype
-          :current_worker_rev
-        when :expire_time
-          nil
-        else
-          self.send(field)        
-        end
+      template = {
+        :tasktype    => :current_worker_rev,
+        :expire_time => nil
+      }
+      fields.collect do |field|
+        template[field] || self.send(field)
       end
     end
   end
 
   class WorkerStatusMessage < Skynet::Message
-    self.fields = {
-      0  => :tasktype,
-      1  => :tasksubtype,
-      2  => :worker_id,
-      3  => :hostname,
-      4  => :process_id,
-      5  => :job_id,
-      6  => :task_id,
-      7  => :iteration,
-      8  => :name,
-      9  => :map_or_reduce,
-      10 => :started_at,
-      11 => :version,
-      12 => :processed
-    }
-    self.fields.values.each { |method| attr_accessor method }
+    self.fields = [
+      :tasktype,
+      :tasksubtype,
+      :worker_id,
+      :hostname,
+      :process_id,
+      :job_id,
+      :task_id,
+      :iteration,
+      :name,
+      :map_or_reduce,
+      :started_at,
+      :version,
+      :processed,
+      :queue_id
+    ]
+    self.fields.each { |method| attr_accessor method }
 
     def initialize(opts)
       super
@@ -336,31 +335,27 @@ class Skynet
     end
     
     def self.worker_status_template(opts)
-      fields.keys.sort.collect do |key|      
-        case fields[key]
-        when :tasktype : :status
-        when :tasksubtype : :worker
-        when :hostname : opts[:hostname]
-        when :process_id : opts[:process_id]
-        else 
-          nil
-        end
+      template = {
+        :tasktype    => :status,
+        :tasksubtype => :worker,
+        :hostname    => opts[:hostname],
+        :process_id  => opts[:process_id]        
+      }
+      fields.collect do |field|
+        template[field]
       end
     end
     
     def self.all_workers_template(hostname=nil)
-      fields.keys.sort.collect do |key|      
-        case fields[key]
-        when :tasktype : :status
-        when :tasksubtype : :worker
-        when :hostname 
-          hostname if hostname
-        else 
-          nil
-        end
+      template = {
+        :tasktype    => :status,
+        :tasksubtype => :worker,
+        :hostname    => hostname,
+      }
+      fields.collect do |field|
+        template[field]
       end
-    end
-    
-  end
+    end    
+  end # class WorkerStatusMessage
 
 end