skynet 0.9.1 → 0.9.2

data/lib/skynet/skynet_message_queue.rb

@@ -12,15 +12,14 @@ class Skynet
   class RequestExpiredError < Skynet::Error
   end
 
+  # This class is the interface to the Skynet Message Queue.   
   class MessageQueue
 
     include SkynetDebugger
 
     require 'forwardable'    
     extend Forwardable    
-    
-    # require 'skynet_message'                            
-    
+        
     def self.adapter
       Object.module_eval(Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER], __FILE__, __LINE__).adapter
     end
@@ -34,8 +33,26 @@ class Skynet
       mq
     end
 
-    def message_fields
-      Skynet::Message.fields
+    # Is this version still active in the queue?
+    def version_active?(version,queue_id=0)
+      mq.version_active?(version,queue_id)
+    end
+
+    # Retrieves the current worker version
+    def get_worker_version
+      mq.get_worker_version
+    end
+    
+    # Sets the current worker version (causing workers to restart)
+    def set_worker_version(version)
+      mq.set_worker_version(version)
+    end
+    
+    # Increments the current worker version (causing workers to restart)
+    def increment_worker_version
+      newver = self.get_worker_version + 1
+      self.set_worker_version(newver)
+      newver
     end
 
     def mq
@@ -43,14 +60,16 @@ class Skynet
     end
                
     def_delegators :mq, :take_next_task, :write_message, :take_result, :write_error, :write_result,
-                   :list_tasks, :list_results,
+                   :list_tasks, :list_results, :stats,
                    :clear_outstanding_tasks, :clear_outstanding_results,
-                   :take_worker_status, :write_worker_status, :read_all_worker_statuses, :clear_worker_status,
-                   :get_worker_version, :set_worker_version, :stats
-                           
-
+                   :take_worker_status, :write_worker_status, :read_all_worker_statuses, :clear_worker_status
                    
 
+          
+   def message_fields
+     Skynet::Message.fields
+   end
+
     def print_stats
       "TAKEN TASKS: #{list_tasks(1).size}, UNTAKEN_TASKS: #{list_tasks(0).size} RESULTS: #{list_results.size}"
     end
@@ -59,12 +78,6 @@ class Skynet
       list_tasks + list_results
     end
 
-    def increment_worker_version
-      newver = self.get_worker_version + 1
-      self.set_worker_version(newver)
-      newver
-    end
-
     def ansi_clear
       puts "\033[2J\033[H"
     end

data/lib/skynet/skynet_partitioners.rb

@@ -0,0 +1,92 @@
+class Skynet
+                                       # Collection of partitioning utilities
+  class Partitioners
+    include SkynetDebugger
+
+    # Split one block of data into partitions
+    #
+    class SimplePartitionData < Partitioners
+
+      def self.reduce_partition(data, partitions)    
+        partitioned_data = Array.new
+
+        # If data size is significantly greater than the number of desired
+        # partitions, we can divide the data roughly but the last partition
+        # may be smaller than the others.
+        #      
+        return data if (not data) or data.empty?
+    
+        if partitions >= data.length
+          data.each do |datum|
+           partitioned_data << [datum]
+          end
+        elsif (data.length >= partitions * 2)
+          # Use quicker but less "fair" method
+          size = data.length / partitions
+
+          if (data.length % partitions != 0)
+            size += 1 # Last slice of leftovers
+          end
+
+          (0..partitions - 1).each do |i|
+            partitioned_data[i] = data[i * size, size]
+          end
+        else
+          # Slower method, but partitions evenly
+          partitions = (data.size < partitions ? data.size : partitions)
+          (0..partitions - 1).each { |i| partitioned_data[i] = Array.new }
+  
+          data.each_with_index do |datum, i|
+            partitioned_data[i % partitions] << datum
+          end
+        end
+  
+        partitioned_data
+      end
+    end
+
+
+    class RecombineAndSplit < Partitioners
+      # Tries to be smart about what kind of data its getting, whether array of arrays or array of arrays of arrays.
+      #
+      def self.reduce_partition(post_map_data,new_partitions)
+        return post_map_data unless post_map_data.is_a?(Array) and (not post_map_data.empty?) and post_map_data.first.is_a?(Array)
+        if not post_map_data.first.first.is_a?(Array)
+          partitioned_data = post_map_data.flatten
+        else
+          partitioned_data = post_map_data.inject(Array.new) do |data,part| 
+            data += part
+          end    
+        end    
+        partitioned_data = Skynet::Partitioners::SimplePartitionData.reduce_partition(partitioned_data, new_partitions)
+        debug "POST PARTITIONED DATA", partitioned_data
+        partitioned_data
+      end
+    end
+    
+    class ArrayDataSplitByFirstEntry < Partitioners
+      # Smarter partitioner for array data, generates simple sum of array[0]
+      # and ensures that all arrays sharing that key go into the same partition.
+      #
+      def self.reduce_partition(partitioned_data, new_partitions)
+        partitions = Array.new
+        (0..new_partitions - 1).each { |i| partitions[i] = Array.new }
+
+        partitioned_data.each do |partition|
+          partition.each do |array|
+            next unless array.class == Array and array.size == 2
+            if array[0].kind_of?(Fixnum)
+              key = array[0]
+            else
+              key = 0
+              array[0].each_byte { |c| key += c }
+            end
+            partitions[key % new_partitions] << array
+          end
+        end
+        partitions
+      end
+    end
+
+  end
+end

data/lib/skynet/skynet_ruby_extensions.rb

@@ -7,7 +7,7 @@ module Enumerable
       data = self
     end
     jobopts = {
-      :map_tasks              => 20000,
+      :mappers                => 20000,
       :map_data               => data,
       :name                   => "#{klass} Enumerable MASTER",
       :map_name               => "#{klass} Enumerable MAP",
@@ -15,8 +15,7 @@ module Enumerable
       :map_timeout            => 3600,
       :reduce_timeout         => 3600,
       :master_timeout         => 3600,
-      :master_result_timeout  => 3600,
-      :async                  => false
+      :master_result_timeout  => 3600
     }                                
 
     jobopts[:map_reduce_class] = klass.to_s if klass
@@ -27,7 +26,7 @@ module Enumerable
     end                               
     
     if block_given? or not jobopts[:async]
-      job = Skynet::Job.new(jobopts)
+      job = Skynet::Job.new(jobopts.merge(:local_master => true))
     else
       job = Skynet::AsyncJob.new(jobopts)
     end

data/lib/skynet/skynet_task.rb

@@ -1,15 +1,12 @@
 class Skynet
   class Task
-    
     include SkynetDebugger
-        
-    # require 'ostruct'
 
-    class ConstructorError < StandardError
-    end
+    class ConstructorError < StandardError; end
+    class TimeoutError     < StandardError; end
     
-    attr_reader :data, :process, :result, :map_or_reduce
-    attr_accessor :name, :tuple, :result_timeout
+    attr_reader :data, :process, :result, :map_or_reduce, :marshalable
+    attr_accessor :name, :result_timeout, :retry
     
     @@log = nil
     
@@ -17,6 +14,31 @@ class Skynet
       "TASK"
     end
     
+    def self.master_task(job)
+      options = {
+        :async        => false,
+        :local_master => true,
+        :map_name     => job.map_name || job.name,
+        :reduce_name  => job.reduce_name || job.name,          
+      }
+      Skynet::Job::FIELDS.each do |field|
+        next if options.has_key?(field)
+        options[field] = job.send(field) if job.send(field)
+      end
+      
+      master_job = Skynet::Job.new(options)
+      
+      self.new(
+        :task_id        => master_job.task_id, 
+        :data           => nil, 
+        :process        => master_job.to_h, 
+        :map_or_reduce  => :master,
+        :name           => master_job.name,
+        :result_timeout => master_job.master_timeout,
+        :retry          => master_job.master_retry || Skynet::CONFIG[:DEFAULT_MASTER_RETRY]
+      )
+    end
+    
     def initialize(opts = {})
       unless opts[:task_id] and opts[:process] and opts[:map_or_reduce]
         raise ConstructorError.new("Must provide task_id, process and map_or_reduce")      
@@ -28,6 +50,7 @@ class Skynet
       @name           = opts[:name]
       @map_or_reduce  = opts[:map_or_reduce]
       @result_timeout = opts[:result_timeout]
+      @retry          = opts[:retry]
     end
     
     def process=(process)
@@ -53,22 +76,41 @@ class Skynet
       @task_id.to_i
     end
     
-    def run
-      debug "running task #{name} task_id:#{task_id} MorR:#{map_or_reduce} PROCESS CLASS: #{@process.class}"
+    def run(iteration=nil)
+      info "running task #{name} TIMEOUT: #{result_timeout} task_id:#{task_id} MorR:#{map_or_reduce} PROCESS CLASS: #{@process.class}"
       begin
-        if @process.class == Proc
-          debug " - #{@map_or_reduce} using Proc"
-          @process.call @data
-        elsif @map_or_reduce == :master
-          debug " - as master"
-          job = Skynet::Job.new(@process)
-          job.run
-        elsif @process.class == String
-          debug " - #{@map_or_reduce} using class #{@process}"
-          @process.constantize.send(@map_or_reduce,@data)
+        Timeout::timeout(@result_timeout) do
+          if @process.class == Proc
+            debug " - #{@map_or_reduce} using Proc"
+            @process.call @data
+          elsif @map_or_reduce == :master
+            debug " - as master"
+            job = Skynet::Job.new(@process)
+            job.run
+          elsif @process.class == String
+            debug " - #{@map_or_reduce} using class #{@process}"
+            @process.constantize.send(@map_or_reduce,@data)
+          end
+        end
+      rescue Timeout::Error => e
+        # ==========
+        # = XXX NEWSFEED HACK
+        # = I'm printing the data hash, but that hash has all this shit added to it after runing through newsfeed.
+        # = It's actually nice to be able to see what was added, but sometimes its too much data.
+        # = Though the handy part will be adding instrumentation to the event_hash and seeing it onyl during a timeout.
+        # ==========
+        
+        if @data.is_a?(Array) and @data.first.is_a?(Hash)          
+          @data.each {|h|h.delete(:event_object)}                      
         end
+        raise TimeoutError.new("TASK TIMED OUT! #{name} IT:[#{iteration}] timeout:#{@result_timeout} #{e.inspect} DATA: #{@data.inspect} #{e.backtrace.join("\n")}")
+
+      # ==========
+      # = XXX This rescue block is probably not necessary.  Just for debugging for now. =
+      # ==========
       rescue Exception => e
         error "Error running task #{e.inspect} TASK:", self, e.backtrace.join("\n")
+        raise e
       end
     end
     

data/lib/skynet/skynet_tuplespace_server.rb

@@ -1,7 +1,5 @@
 #!/usr/bin/env ruby -w
 
-# Rinda RingServer
-
 require 'rinda/ring'
 require 'rinda/tuplespace'
 require 'rubygems'

data/lib/skynet/skynet_worker.rb

@@ -5,35 +5,32 @@ class Skynet
     include Skynet::GuidGenerator
 
     RETRY_TIME = 2
-    VERSION_CHECK_DELAY = 5
-    MAX_MEMORY = 500
+    Skynet::CONFIG[:WORKER_VERSION_CHECK_DELAY] ||= 30
+
+    Skynet::CONFIG[:WORKER_MAX_MEMORY] ||= 500
+
     MEMORY_CHECK_DELAY = 30
     MANAGER_PING_INTERVAL = 60
 
     attr_accessor :message,:task, :mq, :processed
-    attr_reader :worker_id, :worker_info, :worker_type
+    attr_reader :worker_id, :worker_info, :worker_type, :queue_id
     
-    class Error < StandardError
-    end
-
-    class RespawnWorker < Skynet::Error
-    end
-
-    class ConnectionFailure < Skynet::Error
-    end
-    
-    class NoManagerError < Skynet::Error
-    end
+    class Error             < StandardError; end
+    class RespawnWorker     < Skynet::Error; end
+    class ConnectionFailure < Skynet::Error; end
+    class NoManagerError    < Skynet::Error; end
 
     def self.debug_class_desc
       "WORKER-#{$$}"
     end
     
-    def initialize(worker_type=:any)
+    def initialize(worker_type, options = {})                
       @worker_id    = get_unique_id(1).to_i
-      @mq           = Skynet::MessageQueue.new
       @worker_type  = worker_type.to_sym
+      @queue_id     = options[:queue_id] || 0
       @processed    = 0
+      @mq           = Skynet::MessageQueue.new
+
       debug "THIS WORKER TAKES #{worker_type}"
 
       @worker_info = { 
@@ -41,8 +38,9 @@ class Skynet
         :process_id   => process_id,
         :worker_type  => payload_type, 
         :worker_id    => worker_id,
-        :version      => mq.get_worker_version
-      }
+        :version      => mq.get_worker_version,
+      }                           
+      @worker_info.merge!(options)
     end
 
     def process_id
@@ -68,14 +66,14 @@ class Skynet
           @curver = 1
         end
       else    
-        if Time.now < (@verchecktime + VERSION_CHECK_DELAY)
+        if Time.now < (@verchecktime + Skynet::CONFIG[:WORKER_VERSION_CHECK_DELAY])
           return false
         else                    
           @verchecktime = Time.now
           begin
             newver = mq.get_worker_version
             # debug "CURVER #{@curver} NEWVER: #{newver}"
-            if newver != @curver
+            if newver != @curver and not mq.version_active?(@curver, queue_id)              
               info "RESTARTING WORKER ON PID #{$$}"
               return true
             end
@@ -138,42 +136,48 @@ class Skynet
       return nil if worker_type == :any
       return worker_type      
     end
+    
+    def interrupt
+      if @die
+        exit
+      else
+        @die = true
+      end        
+    end
           
     def start
       exceptions = 0
-      conerror = 0
-      @curver = nil
+      conerror   = 0
+      @curver    = nil
+      
       # setup signal handlers for manager
       Signal.trap("HUP")  { @respawn = true }
-      Signal.trap("TERM") do
-        if @die
-          exit
-        else
-          @die = true
-        end        
-      end
-      Signal.trap("INT")  { @die = true }
-    
+      Signal.trap("TERM") { interrupt       }
+      Signal.trap("INT")  { @die = true     }
+      
       raise Skynet::Worker::RespawnWorker.new if new_version_respawn?
         
-      info "STARTING WORKER @ VER #{@curver} (#{@worker_type})"
+      info "STARTING WORKER @ VER #{@curver} (#{@worker_type}) QUEUE_ID: #{queue_id}"
 
       notify_worker_started
 
       message = nil
-      task = nil
+      task    = nil
 
       loop do
         message = nil      
         begin               
+          if Skynet::CONFIG[:WORKER_MAX_PROCESSED] and Skynet::CONFIG[:WORKER_MAX_PROCESSED] > 0 and @processed >= Skynet::CONFIG[:WORKER_MAX_PROCESSED]
+            raise Skynet::Worker::RespawnWorker.new("WORKER OVER MAX MEM AT: #{get_memory_size} MAX: #{Skynet::CONFIG[:WORKER_MAX_MEMORY]}")
+          end
           if @die             
             exit                                  
           elsif @respawn
-            raise Skynet::Worker::RespawnWorker.new 
+            raise Skynet::Worker::RespawnWorker.new()
           end
 
           if local_mem = max_memory_reached?
-            raise Skynet::Worker::RespawnWorker.new("WORKER OVER MAX MEM AT: #{local_mem} MAX: #{MAX_MEMORY}")
+            raise Skynet::Worker::RespawnWorker.new("WORKER OVER MAX MEM AT: #{local_mem} MAX: #{Skynet::CONFIG[:WORKER_MAX_MEMORY]}")
           end
           
           if conerror > 0
@@ -186,7 +190,7 @@ class Skynet
           # 
           # debug "LOOK FOR WORK USING TEMPLATE", Skynet::Message.task_template(@curver)
           # message = Skynet::Message.new(mq.take(Skynet::Message.task_template(@curver),0.00001))
-          message = mq.take_next_task(@curver,0.00001,payload_type)
+          message = mq.take_next_task(@curver, 0.00001, payload_type, queue_id)
 
           next unless message.respond_to?(:payload)
 
@@ -210,29 +214,34 @@ class Skynet
             :name          => message.name, 
             :map_or_reduce => task.map_or_reduce
           })
-          result = task.run
+          result = task.run(message.iteration)
 
           info "STEP 5 GOT RESULT FROM RUN TASK #{message.name} jobid: #{message.job_id} taskid: #{task.task_id}"
           debug "STEP 5.1 RESULT DATA:", result
 
-          ## XXX need better result timeout
           result_message = mq.write_result(message,result,task.result_timeout)
           info "STEP 6 WROTE RESULT MESSAGE #{message.name} jobid: #{message.job_id} taskid: #{task.task_id}"
           # debug "STEP 6.1 RESULT_MESSAGE:", result_message
           notify_task_complete          
+
+        rescue Skynet::Task::TimeoutError => e
+          error "Task timed out while executing #{e.inspect} #{e.backtrace.join("\n")}"
+          next
+
         rescue Skynet::Worker::RespawnWorker => e  
-          info "Respawning and taking worker status"
+          info "Respawning and taking worker status #{e.message}"
           notify_worker_stop
           raise e          
+
         rescue Skynet::RequestExpiredError => e
-          # debug "request expired"
           if new_version_respawn?
             notify_worker_stop
-            raise Skynet::Worker::RespawnWorker.new
+            manager = DRbObject.new(nil, Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
+            manager.restart_worker($$) if manager
           end
           sleep 1
-          # debug "WORKER [#{$$}] LOOPING AGAIN"
           next
+
         rescue Skynet::ConnectionError, DRb::DRbConnError => e
           conerror += 1
           retry_time = conerror > 6 ? RETRY_TIME * 3 : RETRY_TIME
@@ -245,6 +254,7 @@ class Skynet
             raise e 
           end
           next
+
         rescue NoManagerError => e
           fatal e.message
           break          
@@ -253,7 +263,9 @@ class Skynet
           notify_worker_stop
           break
         rescue Exception => e
-          error "#{e.inspect} #{e.backtrace.join("\n")}"
+          error "skynet_worker.rb:#{__LINE__} #{e.inspect} #{e.backtrace.join("\n")}"
+          exceptions += 1
+          break if exceptions > 1000
           #mq.take(@next_worker_message.task_template,0.0005) if message
           if message
             mq.write_error(message,"#{e.inspect} #{e.backtrace.join("\n")}",(task.respond_to?(:result_timeout) ? task.result_timeout : 200))
@@ -279,7 +291,7 @@ class Skynet
       elsif Time.now > (@memchecktime + MEMORY_CHECK_DELAY)
         @memchecktime = Time.now
         local_mem = get_memory_size.to_i
-        return local_mem if local_mem > MAX_MEMORY
+        return local_mem if local_mem > Skynet::CONFIG[:WORKER_MAX_MEMORY]
       else
         false
       end
@@ -346,9 +358,22 @@ class Skynet
             raise Skynet::Error.new("#{v} is not a valid worker_type")
           end
         end
+        opt.on('-q', '--queue QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v| 
+          options[:queue] = v
+        end               
+        opt.on('-i', '--queue_id queue_id', 'Which queue should these workers use (default 0).') do |v| 
+          options[:queue_id] = v.to_i
+        end               
         opt.parse!(ARGV)
       end
 
+      if options[:queue]
+        if options[:queue_id]
+          raise Skynet::Error.new("You may either provide a queue_id or a queue, but not both.")
+        end
+        options[:queue_id] = config.queue_id_by_name(options[:queue])
+      end
+
       options[:required_libs].each do |adlib|
         begin
           require adlib
@@ -358,22 +383,19 @@ class Skynet
         end
       end        
 
-      # worker_script_path = (Skynet::CONFIG[:WORKER_SCRIPT_PATH] || File.dirname(__FILE__)) << "/skynet_worker"
-
-      debug "WORKER STARTING WORKER_TYPE?:#{options[:worker_type]}"
+      debug "WORKER STARTING WORKER_TYPE?:#{options[:worker_type]}. QUEUE: #{Skynet::Config.new.queue_name_by_id(options[:queue_id])}"
 
       begin                               
-        worker = Skynet::Worker.new(options[:worker_type])
+        worker = Skynet::Worker.new(options[:worker_type], options)
         worker.start
       rescue Skynet::Worker::NoManagerError => e
         fatal e.message
         exit          
       rescue Skynet::Worker::RespawnWorker => e
-        warn "WORKER #{$$} SCRIPT CAUGHT RESPAWN.  RESTARTING"
-        cmd = "RAILS_ENV=#{RAILS_ENV} ruby #{Skynet::CONFIG[:LAUNCHER_PATH]} --worker_type=#{options[:worker_type]}"
+        warn "WORKER #{$$} SCRIPT CAUGHT RESPAWN.  RESTARTING #{e.message}"
+        cmd = "ruby #{Skynet::CONFIG[:LAUNCHER_PATH]} --worker_type=#{options[:worker_type]} --queue_id=#{options[:queue_id]}"
         cmd << "-r #{options[:required_libs].join(' -r ')}" if options[:required_libs] and not options[:required_libs].empty?
         pid = fork_and_exec(cmd)
-        warn "parent_pid: #{$$}, child_pid: #{pid}"
         exit
       rescue SystemExit
         info "WORKER #{$$} EXITING GRACEFULLY"