skynet 0.9.2 → 0.9.3

data/lib/skynet/skynet_message.rb

@@ -310,6 +310,7 @@ class Skynet
   end
 
   class WorkerStatusMessage < Skynet::Message
+# I'd love to rename tasktype and tasksubtype to something more reasonable
     self.fields = [
       :tasktype,
       :tasksubtype,
@@ -330,13 +331,11 @@ class Skynet
 
     def initialize(opts)
       super
-      self.tasktype    = :status
       self.tasksubtype = :worker
     end
     
     def self.worker_status_template(opts)
       template = {
-        :tasktype    => :status,
         :tasksubtype => :worker,
         :hostname    => opts[:hostname],
         :process_id  => opts[:process_id]        
@@ -348,7 +347,6 @@ class Skynet
     
     def self.all_workers_template(hostname=nil)
       template = {
-        :tasktype    => :status,
         :tasksubtype => :worker,
         :hostname    => hostname,
       }

data/lib/skynet/skynet_message_queue.rb

@@ -12,27 +12,48 @@ class Skynet
   class RequestExpiredError < Skynet::Error
   end
 
-  # This class is the interface to the Skynet Message Queue.   
+  # This class is the interface to the Skynet Message Queue.
   class MessageQueue
 
     include SkynetDebugger
 
-    require 'forwardable'    
-    extend Forwardable    
-        
+    require 'forwardable'
+    extend Forwardable
+
     def self.adapter
-      Object.module_eval(Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER], __FILE__, __LINE__).adapter
+      adapter_class.constantize.adapter
     end
 
-    def initialize(message_queue_proxy_class=Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER])
-      if message_queue_proxy_class.is_a?(String)
-        @message_queue_proxy_class = Object.module_eval(message_queue_proxy_class, __FILE__, __LINE__)
-      else
-        @message_queue_proxy_class = message_queue_proxy_class
+    def self.adapter_class
+      Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER]
+    end
+    
+    def self.start_or_connect(adapter_class = self.adapter_class)
+      begin
+        mq = new
+      rescue Skynet::ConnectionError
+        if self.adapter == :tuplespace
+          pid = fork do
+            exec("skynet_tuplespace_server start")
+          end
+          sleep 5
+        end
+        new
       end
+    end
+    
+    def initialize(message_queue_proxy_class=Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER])
       mq
     end
 
+    def self.message_queue_proxy_class
+      adapter_class.constantize
+    end
+
+    def message_queue_proxy_class
+      @message_queue_proxy_class ||= self.class.message_queue_proxy_class
+    end
+
     # Is this version still active in the queue?
     def version_active?(version,queue_id=0)
       mq.version_active?(version,queue_id)
@@ -42,12 +63,12 @@ class Skynet
     def get_worker_version
       mq.get_worker_version
     end
-    
+
     # Sets the current worker version (causing workers to restart)
     def set_worker_version(version)
       mq.set_worker_version(version)
     end
-    
+
     # Increments the current worker version (causing workers to restart)
     def increment_worker_version
       newver = self.get_worker_version + 1
@@ -56,16 +77,18 @@ class Skynet
     end
 
     def mq
-      @mq ||= @message_queue_proxy_class.new
+      @mq ||= message_queue_proxy_class.new(
+        :use_ringserver   => Skynet::CONFIG[:TS_USE_RINGSERVER],
+        :ringserver_hosts => Skynet::CONFIG[:TS_SERVER_HOSTS],
+        :drburi           => Skynet::CONFIG[:TS_DRBURI]
+      )
     end
-               
+
     def_delegators :mq, :take_next_task, :write_message, :take_result, :write_error, :write_result,
-                   :list_tasks, :list_results, :stats,
-                   :clear_outstanding_tasks, :clear_outstanding_results,
-                   :take_worker_status, :write_worker_status, :read_all_worker_statuses, :clear_worker_status
-                   
+                   :list_tasks, :list_results, :clear_outstanding_tasks, :clear_outstanding_results, :stats
+
+
 
-          
    def message_fields
      Skynet::Message.fields
    end

data/lib/skynet/skynet_partitioners.rb

@@ -51,14 +51,16 @@ class Skynet
       #
       def self.reduce_partition(post_map_data,new_partitions)
         return post_map_data unless post_map_data.is_a?(Array) and (not post_map_data.empty?) and post_map_data.first.is_a?(Array)
-        if not post_map_data.first.first.is_a?(Array)
-          partitioned_data = post_map_data.flatten
-        else
+        ### Why did I do this?  It breaks badly.
+        # if not post_map_data.first.first.is_a?(Array)
+        #   partitioned_data = post_map_data.flatten
+        # else
           partitioned_data = post_map_data.inject(Array.new) do |data,part| 
             data += part
           end    
-        end    
+        # end    
         partitioned_data = Skynet::Partitioners::SimplePartitionData.reduce_partition(partitioned_data, new_partitions)
+        debug "POST PARTITIONED DATA_SIZE", partitioned_data.size
         debug "POST PARTITIONED DATA", partitioned_data
         partitioned_data
       end
@@ -68,20 +70,22 @@ class Skynet
       # Smarter partitioner for array data, generates simple sum of array[0]
       # and ensures that all arrays sharing that key go into the same partition.
       #
-      def self.reduce_partition(partitioned_data, new_partitions)
-        partitions = Array.new
+      def self.reduce_partition(post_map_data, new_partitions)
+        partitions = []
         (0..new_partitions - 1).each { |i| partitions[i] = Array.new }
-
-        partitioned_data.each do |partition|
-          partition.each do |array|
-            next unless array.class == Array and array.size == 2
+        cnt = 0
+        post_map_data.each do |partition|
+          partition.each do |array|            
+            next unless array.is_a?(Array) and array.size >= 2
             if array[0].kind_of?(Fixnum)
-              key = array[0]
-            else
-              key = 0
-              array[0].each_byte { |c| key += c }
+              key = array[0] % new_partitions
+            elsif array[0].kind_of?(String)
+              key = array[0].sum % new_partitions
+            else                 
+              cnt += 1
+              key = cnt % new_partitions
             end
-            partitions[key % new_partitions] << array
+            partitions[key] << array
           end
         end
         partitions

data/lib/skynet/skynet_ruby_extensions.rb

@@ -32,4 +32,22 @@ module Enumerable
     end
     job.run    
   end  
+end
+
+class String
+  ### THIS IS TAKEN DIRECTLY FROM ActiveSupport::Inflector
+  # Constantize tries to find a declared constant with the name specified
+  # in the string. It raises a NameError when the name is not in CamelCase
+  # or is not initialized.
+  #
+  # Examples
+  #   "Module".constantize #=> Module
+  #   "Class".constantize #=> Class
+  def constantize
+    unless /\A(?:::)?([A-Z]\w*(?:::[A-Z]\w*)*)\z/ =~ self
+      raise NameError, "#{camel_cased_word.inspect} is not a valid constant name!"
+    end
+
+    Object.module_eval("::#{$1}", __FILE__, __LINE__)
+  end
 end

data/lib/skynet/skynet_tuplespace_server.rb

@@ -49,28 +49,31 @@ class Skynet
   class Job
   end
 
-  class Server
+  class TuplespaceServer
 
-    def initialize(options)
-      log = Logger.new(options[:logfile], 'weekly')
-      log.level = Object.module_eval("#{"Logger::" + options[:loglevel].upcase}", __FILE__, __LINE__)
-      log.info "STARTING SKYNET SERVER ON PORT: #{options[:port]} Logging to #{options[:logfile]}"
+    def self.start(options)
+      options[:port] ||= 7647
+      log = Logger.new(options[:logfile])
+      log.level = Object.module_eval("#{"Logger::" + options[:loglevel].upcase}", __FILE__, __LINE__) if options[:loglevel]
+      log.info "STARTING TUPLESPACE SERVER ON PORT: #{options[:port]} Logging to #{options[:logfile]}"
 
       # Create a TupleSpace to hold named services, and start running
+      ts = Rinda::TupleSpace.new
       begin
-        ts = Rinda::TupleSpace.new
+        if options[:use_ringserver] and options[:port]
+          DRb.start_service
+          tuple = [:name,:TupleSpace, ts, 'Tuple Space']
+          renewer = Rinda::SimpleRenewer.new
+          ring_ts = Rinda::TupleSpace.new
+          ring_ts.write(tuple, renewer)
+
+          server = Rinda::RingServer.new(ring_ts, options[:port])
+        end
         if options[:drburi]
           DRb.start_service(options[:drburi], ts)          
-        else
-          DRb.start_service
         end
-        tuple = [:name,:TupleSpace, ts, 'Tuple Space']
-        renewer = Rinda::SimpleRenewer.new
-        ring_ts = Rinda::TupleSpace.new
-        ring_ts.write(tuple, renewer)
-
-        server = Rinda::RingServer.new(ring_ts, options[:port])
         DRb.thread.join
+      rescue SystemExit, Interrupt
       rescue Exception, RuntimeError => e
         log.fatal "Couldn't start Skynet Server #{e.inspect}"
       end

data/lib/skynet/skynet_worker.rb

@@ -1,7 +1,7 @@
-class Skynet  
+class Skynet
   class Worker
 
-    include SkynetDebugger    
+    include SkynetDebugger
     include Skynet::GuidGenerator
 
     RETRY_TIME = 2
@@ -14,7 +14,7 @@ class Skynet
 
     attr_accessor :message,:task, :mq, :processed
     attr_reader :worker_id, :worker_info, :worker_type, :queue_id
-    
+
     class Error             < StandardError; end
     class RespawnWorker     < Skynet::Error; end
     class ConnectionFailure < Skynet::Error; end
@@ -23,57 +23,59 @@ class Skynet
     def self.debug_class_desc
       "WORKER-#{$$}"
     end
-    
-    def initialize(worker_type, options = {})                
+
+    def initialize(worker_type, options = {})
       @worker_id    = get_unique_id(1).to_i
       @worker_type  = worker_type.to_sym
       @queue_id     = options[:queue_id] || 0
       @processed    = 0
+      @in_process   = false
       @mq           = Skynet::MessageQueue.new
 
       debug "THIS WORKER TAKES #{worker_type}"
 
-      @worker_info = { 
+      @worker_info = {
+        :tasktype     => worker_type,
         :hostname     => hostname,
         :process_id   => process_id,
-        :worker_type  => payload_type, 
+        :worker_type  => payload_type,
         :worker_id    => worker_id,
         :version      => mq.get_worker_version,
-      }                           
+      }
       @worker_info.merge!(options)
     end
 
     def process_id
       $$
     end
-    
+
     def hostname
       @machine_name ||= Socket.gethostname
-    end       
-    
+    end
+
     def version
       @curver
     end
-        
+
     def new_version_respawn?
        if !@verchecktime
         @verchecktime = Time.now
-        begin 
+        begin
           @curver = mq.get_worker_version
           debug "FINDING INITIAL VER #{@curver}"
         rescue  Skynet::RequestExpiredError => e
           warn "NO INITIAL VER IN MQ using 1"
           @curver = 1
         end
-      else    
+      else
         if Time.now < (@verchecktime + Skynet::CONFIG[:WORKER_VERSION_CHECK_DELAY])
           return false
-        else                    
+        else
           @verchecktime = Time.now
           begin
             newver = mq.get_worker_version
             # debug "CURVER #{@curver} NEWVER: #{newver}"
-            if newver != @curver and not mq.version_active?(@curver, queue_id)              
+            if newver != @curver and not mq.version_active?(@curver, queue_id)
               info "RESTARTING WORKER ON PID #{$$}"
               return true
             end
@@ -85,19 +87,10 @@ class Skynet
         end
       end
       return false
-    end                     
-
-
-    def take_worker_status
-      begin               
-        mq.take_worker_status(@worker_info,0.00001)
-      rescue Skynet::RequestExpiredError, Skynet::QueueTimeout => e
-        error "Couldnt take worker status for #{hostname} pid: #{process_id}"
-      end
     end
-    
+
     def notify_worker_started
-      mq.write_worker_status(
+      write_worker_status(
         @worker_info.merge({
           :name       => "waiting for #{@worker_type}",
           :processed  => 0,
@@ -105,17 +98,19 @@ class Skynet
         })
       )
     end
-    
+
     def notify_task_begun(task)
       task[:processed] = @processed
       task[:started_at] = Time.now.to_i
-      mq.write_worker_status(@worker_info.merge(task))
+      @in_process = true      
+      write_worker_status(@worker_info.merge(task))
     end
-    
+
     def notify_task_complete
       @processed += 1
+      @in_process = false
 
-      mq.write_worker_status(
+      write_worker_status(
         @worker_info.merge({
           :task_id       => 0,
           :job_id        => 0,
@@ -125,39 +120,74 @@ class Skynet
           :started_at    => Time.now.to_i
         })
       )
-    end 
-    
+    end
+
     def notify_worker_stop
       info "Worker #{process_id} stopping..."
-      take_worker_status
-    end   
+      write_worker_status(
+      @worker_info.merge({
+        :task_id       => 0,
+        :job_id        => 0,
+        :name          => "waiting for #{@worker_type}",
+        :processed     => @processed,
+        :process_id    => nil,
+        :map_or_reduce => nil,
+        :started_at    => Time.now.to_i
+        })
+      )
+    end           
+    
+    def manager_send(method,*args)
+      begin
+        manager.send(method,*args)
+      rescue DRb::DRbConnError, Errno::ECONNREFUSED  => e
+        error "Worker could not connect to manager to call #{method} on manager #{e.inspect}"
+      rescue Exception  => e
+        error "Worker could not connect call #{method} on manager #{e.inspect} args:", args
+      end
+    end
+
+    def write_worker_status(status)
+      manager_send(:worker_notify,status)
+    end                              
+    
+    def manager
+      Skynet::Manager.get
+    end
     
     def payload_type
       return nil if worker_type == :any
-      return worker_type      
+      return worker_type
     end
-    
+
     def interrupt
       if @die
         exit
       else
         @die = true
-      end        
+        if not @in_process
+          notify_worker_stop
+          exit
+        end
+      end
     end
-          
+
     def start
       exceptions = 0
       conerror   = 0
       @curver    = nil
-      
+
       # setup signal handlers for manager
-      Signal.trap("HUP")  { @respawn = true }
+      Signal.trap("HUP")  do
+        @respawn = true
+        raise Skynet::Worker::RespawnWorker.new if not @in_process
+      end
       Signal.trap("TERM") { interrupt       }
-      Signal.trap("INT")  { @die = true     }
-      
+      Signal.trap("INT")  { interrupt       }
+
       raise Skynet::Worker::RespawnWorker.new if new_version_respawn?
-        
-      info "STARTING WORKER @ VER #{@curver} (#{@worker_type}) QUEUE_ID: #{queue_id}"
+
+      printlog "STARTING WORKER @ VER:#{@curver} type:#{@worker_type} QUEUE_ID:#{queue_id}"
 
       notify_worker_started
 
@@ -165,13 +195,13 @@ class Skynet
       task    = nil
 
       loop do
-        message = nil      
-        begin               
+        message = nil
+        begin
           if Skynet::CONFIG[:WORKER_MAX_PROCESSED] and Skynet::CONFIG[:WORKER_MAX_PROCESSED] > 0 and @processed >= Skynet::CONFIG[:WORKER_MAX_PROCESSED]
             raise Skynet::Worker::RespawnWorker.new("WORKER OVER MAX MEM AT: #{get_memory_size} MAX: #{Skynet::CONFIG[:WORKER_MAX_MEMORY]}")
           end
-          if @die             
-            exit                                  
+          if @die
+            exit
           elsif @respawn
             raise Skynet::Worker::RespawnWorker.new()
           end
@@ -179,7 +209,7 @@ class Skynet
           if local_mem = max_memory_reached?
             raise Skynet::Worker::RespawnWorker.new("WORKER OVER MAX MEM AT: #{local_mem} MAX: #{Skynet::CONFIG[:WORKER_MAX_MEMORY]}")
           end
-          
+
           if conerror > 0
             @mq = Skynet::MessageQueue.new
             warn "WORKER RECONNECTED AFTER #{conerror} tries"
@@ -187,14 +217,14 @@ class Skynet
           end
 
           # debug "1 START LOOPSSS at VER #{@curver}"
-          # 
+          #
           # debug "LOOK FOR WORK USING TEMPLATE", Skynet::Message.task_template(@curver)
           # message = Skynet::Message.new(mq.take(Skynet::Message.task_template(@curver),0.00001))
           message = mq.take_next_task(@curver, 0.00001, payload_type, queue_id)
 
           next unless message.respond_to?(:payload)
 
-          task = message.payload                          
+          task = message.payload
           error "BAD MESSAGE", task unless task.respond_to?(:map_or_reduce)
 
           info "STEP 2 GOT MESSAGE #{message.name} type:#{task.map_or_reduce}, jobid: #{message.job_id}, taskid:#{message.task_id} it: #{message.iteration}"
@@ -207,11 +237,11 @@ class Skynet
           # task.debug "taking task #{task.task_id} name:#{task.name}..."
 
           info "STEP 4 RUNNING TASK #{message.name} jobid: #{message.job_id} taskid: #{task.task_id}"
-          notify_task_begun({ 
-            :job_id        => message.job_id, 
-            :task_id       => message.task_id, 
-            :iteration     => message.iteration, 
-            :name          => message.name, 
+          notify_task_begun({
+            :job_id        => message.job_id,
+            :task_id       => message.task_id,
+            :iteration     => message.iteration,
+            :name          => message.name,
             :map_or_reduce => task.map_or_reduce
           })
           result = task.run(message.iteration)
@@ -222,22 +252,22 @@ class Skynet
           result_message = mq.write_result(message,result,task.result_timeout)
           info "STEP 6 WROTE RESULT MESSAGE #{message.name} jobid: #{message.job_id} taskid: #{task.task_id}"
           # debug "STEP 6.1 RESULT_MESSAGE:", result_message
-          notify_task_complete          
+          notify_task_complete
 
         rescue Skynet::Task::TimeoutError => e
           error "Task timed out while executing #{e.inspect} #{e.backtrace.join("\n")}"
+          @in_process = false
           next
 
-        rescue Skynet::Worker::RespawnWorker => e  
+        rescue Skynet::Worker::RespawnWorker => e
           info "Respawning and taking worker status #{e.message}"
           notify_worker_stop
-          raise e          
+          raise e
 
         rescue Skynet::RequestExpiredError => e
           if new_version_respawn?
             notify_worker_stop
-            manager = DRbObject.new(nil, Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
-            manager.restart_worker($$) if manager
+            manager_send(:restart_worker,$$)
           end
           sleep 1
           next
@@ -251,15 +281,15 @@ class Skynet
           if conerror > 20
             fatal "TOO MANY RECONNECTION EXCEPTIONS #{e.message}"
             notify_worker_stop
-            raise e 
+            raise e
           end
           next
 
         rescue NoManagerError => e
           fatal e.message
-          break          
+          break
         rescue Interrupt, SystemExit => e
-          warn "Exiting..."
+          info "Exiting..."
           notify_worker_stop
           break
         rescue Exception => e
@@ -274,6 +304,7 @@ class Skynet
             # mq.write_error(message,"ERROR in WORKER [#{$$}] #{e.inspect} #{e.backtrace.join("\n")}")
           end
           # mq.write_error("ERROR in WORKER [#{$$}] #{e.inspect} #{e.backtrace.join("\n")}")
+          @in_process = false
           next
         end
       end
@@ -282,7 +313,7 @@ class Skynet
     @@ok_to_mem_check = false
     @@lastmem = nil
     @@memct = 0
-    
+
     def max_memory_reached?
       return false unless ok_to_mem_check?
        if !@memchecktime
@@ -296,7 +327,7 @@ class Skynet
         false
       end
     end
-    
+
     def find_pid_size(file, format=:notpretty)
       begin
         open(file).each { |line|
@@ -321,49 +352,38 @@ class Skynet
       return false if @@ok_to_mem_check == :notok
       if File.exists?('/proc/self/status')
       @@lastmem ||= get_memory_size.to_i
-      return @@ok_to_mem_check = true         
+      return @@ok_to_mem_check = true
       else
         @@ok_to_mem_check = :notok
-        return false        
-      end      
-    end    
-
-
-      # kinda like system() but gives me back a pid
-    def self.fork_and_exec(command)
-      sleep 0.01  # remove contention on manager drb object
-      log = Skynet::Logger.get
-      info "executing /bin/sh -c \"#{command}\""
-      pid = fork do
-        exec("/bin/sh -c \"#{command}\"")
-        exit
+        return false
       end
-      Process.detach(pid)
-      pid
     end
 
     def self.start(options={})
       options[:worker_type]    ||= :any
       options[:required_libs]  ||= []
-      
+
       OptionParser.new do |opt|
         opt.banner = "Usage: worker [options]"
         opt.on('-r', '--required LIBRARY', 'Include the specified libraries') do |v|
           options[:required_libs] << v
         end
-        opt.on('-ot', '--worker_type WORKERTYPE', "master, task or any") do |v|
+        opt.on('--worker_type=WORKERTYPE', "master, task or any") do |v|
           if ["any","master","task"].include?(v)
             options[:worker_type] = v
           else
             raise Skynet::Error.new("#{v} is not a valid worker_type")
           end
         end
-        opt.on('-q', '--queue QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v| 
+        opt.on('--config=CONFIG_FILE', 'Where to find the skynet.rb config file') do |v|
+          options[:config_file] = File.expand_path(v)
+        end
+        opt.on('--queue=QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v|
           options[:queue] = v
-        end               
-        opt.on('-i', '--queue_id queue_id', 'Which queue should these workers use (default 0).') do |v| 
+        end
+        opt.on('--queue_id=queue_id', 'Which queue should these workers use (default 0).') do |v|
           options[:queue_id] = v.to_i
-        end               
+        end
         opt.parse!(ARGV)
       end
 
@@ -381,21 +401,35 @@ class Skynet
           error "The included lib #{adlib} was not found: #{e.inspect}"
           exit
         end
-      end        
+      end
+
+      options[:config_file] ||= Skynet::CONFIG[:CONFIG_FILE]
+      if options[:config_file]
+        begin
+          require options[:config_file]
+        rescue MissingSourceFile => e
+          error "The config file at #{options[:config_file]} was not found: #{e.inspect}"
+          exit
+        end
+      else
+        error "Config file missing. Please add a config/skynet_config.rb before starting."        
+        exit
+      end
 
       debug "WORKER STARTING WORKER_TYPE?:#{options[:worker_type]}. QUEUE: #{Skynet::Config.new.queue_name_by_id(options[:queue_id])}"
 
-      begin                               
+      begin
         worker = Skynet::Worker.new(options[:worker_type], options)
         worker.start
       rescue Skynet::Worker::NoManagerError => e
         fatal e.message
-        exit          
+        exit
       rescue Skynet::Worker::RespawnWorker => e
         warn "WORKER #{$$} SCRIPT CAUGHT RESPAWN.  RESTARTING #{e.message}"
-        cmd = "ruby #{Skynet::CONFIG[:LAUNCHER_PATH]} --worker_type=#{options[:worker_type]} --queue_id=#{options[:queue_id]}"
+        cmd = "ruby #{Skynet::CONFIG[:LAUNCHER_PATH]} --worker_type=#{options[:worker_type]} --queue_id=#{options[:queue_id]} "
+        cmd << "--config=#{options[:config_file]} "
         cmd << "-r #{options[:required_libs].join(' -r ')}" if options[:required_libs] and not options[:required_libs].empty?
-        pid = fork_and_exec(cmd)
+        pid = Skynet.fork_and_exec(cmd)
         exit
       rescue SystemExit
         info "WORKER #{$$} EXITING GRACEFULLY"
@@ -408,10 +442,10 @@ class Skynet
   end
 end
 
-class ExceptionReport  
-  def initialize(*args)    
-  end                 
-  
+class ExceptionReport
+  def initialize(*args)
+  end
+
   def save
   end
 end