skynet 0.9.1 → 0.9.2

data/lib/skynet/message_queue_adapters/tuple_space.rb

@@ -43,18 +43,20 @@ class Skynet
         @ts = ts
       end
 
-      def take_next_task(curver,timeout=nil,payload_type=nil)
-        message = Skynet::Message.new(take(Skynet::Message.next_task_template(curver,payload_type),timeout))
+      def take_next_task(curver,timeout=nil,payload_type=nil,queue_id=0)
+        message = Skynet::Message.new(take(Skynet::Message.next_task_template(curver,payload_type, queue_id),timeout))
         write_fallback_task(message)
         message
       end
 
       def write_message(message,timeout=nil)
+        timeout ||= message.expiry
         write(message,timeout)
       end
 
       def write_result(message,result=[],timeout=nil)
         result_message = message.result_message(result).to_a
+        timeout ||= result_message.expiry
         write(result_message,timeout)
         take_fallback_message(message)
         result_message
@@ -65,6 +67,7 @@ class Skynet
       end
 
       def write_error(message,error='',timeout=nil)
+        timeout ||= message.expiry
         write(message.error_message(error),timeout)
         take_fallback_message(message)
       end
@@ -99,14 +102,25 @@ class Skynet
         cnt
       end
 
-      def list_tasks(iteration=nil)
-        read_all(Skynet::Message.outstanding_tasks_template(iteration))
+      def list_tasks(iteration=nil,queue_id=0)
+        read_all(Skynet::Message.outstanding_tasks_template(iteration,queue_id))
       end
 
       def list_results
         read_all(Skynet::Message.outstanding_results_template)
       end
 
+      def version_active?(curver=nil, queue_id= 0)
+        return true unless curver
+        begin
+          message_row = read(Skynet::Message.next_task_template(curver, nil, queue_id),0.00001)
+          true
+        rescue Skynet::RequestExpiredError
+          return true if curver.to_i == get_worker_version.to_i
+          false
+        end
+      end
+
       def get_worker_version
         begin
           message = Skynet::WorkerVersionMessage.new(read(Skynet::WorkerVersionMessage.template,0.00001))
@@ -198,22 +212,23 @@ class Skynet
       ###### FALLBACK METHODS
       def write_fallback_task(message)
         return unless USE_FALLBACK_TASKS
-        debug "4 WRITING BACKUP TASK #{message.task_id}", @fallback_worker_message
+        debug "4 WRITING BACKUP TASK #{message.task_id}", message.to_h
         ftm = message.fallback_task_message
-        debug "WRITE FALLBACK TASK", ftm.to_a
+        debug "WRITE FALLBACK TASK", ftm.to_h
         timeout = message.expiry * 8
-        write(ftm,timeout)
+        write(ftm,timeout) unless ftm.iteration == -1
         ftm
       end
 
       def take_fallback_message(message,timeout=0.01)
         return unless USE_FALLBACK_TASKS
+        return if message.retry <= message.iteration
         begin
           # debug "LOOKING FOR FALLBACK TEMPLATE", message.fallback_template
           fb_message = Skynet::Message.new(take(message.fallback_template,timeout))
           # debug "TOOK FALLBACK MESSAGE for TASKID: #{fb_message.task_id}"
         rescue Skynet::RequestExpiredError => e
-          error "Couldn't find expected FALLBACK MESSAGE"
+          error "Couldn't find expected FALLBACK MESSAGE", Skynet::Message.new(message.fallback_template).to_h
         end
       end
       ## END FALLBACK METHODS
@@ -267,28 +282,28 @@ class Skynet
         loop do
           begin
             DRb.start_service
-            if Skynet::CONFIG[:USE_RINGSERVER]
-              Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx] =~ /(.+):(\d+)/
+            if Skynet::CONFIG[:TS_USE_RINGSERVER]
+              Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx] =~ /(.+):(\d+)/
               host = $1
               port = $2.to_i
               @@ts = connect_to_tuple_space(host,port)
             else
-              drburi = Skynet::CONFIG[:TUPLESPACE_DRBURIS].first
+              drburi = Skynet::CONFIG[:TS_DRBURIS].first
               drburi = "druby://#{drburi}" unless drburi =~ %r{druby://}              
               @@ts = get_tuple_space_from_drburi(drburi)
               log.info "#{self} CONNECTED TO #{drburi}"
             end
             return @@ts
           rescue RuntimeError => e
-            if Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx + 1]
-              log.error "#{self} Couldn't connect to #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]} trying #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx+1]}"
+            if Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx + 1]
+              log.error "#{self} Couldn't connect to #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx]} trying #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx+1]}"
               @@curhostidx += 1
               next
             else
-              raise Skynet::ConnectionError.new("Can't find ring finger @ #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")
+              raise Skynet::ConnectionError.new("Can't find ring finger @ #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")
             end
           rescue Exception => e
-            raise Skynet::ConnectionError.new("Error getting tuplespace @ #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")          
+            raise Skynet::ConnectionError.new("Error getting tuplespace @ #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")          
           end
         end
         return @@ts
@@ -296,7 +311,7 @@ class Skynet
     
       def self.connect_to_tuple_space(host,port)
         log.info "#{self} trying to connect to #{host}:#{port}"
-        if Skynet::CONFIG[:USE_RINGSERVER]
+        if Skynet::CONFIG[:TS_USE_RINGSERVER]
           ring_finger = Rinda::RingFinger.new(host,port)
           ring_server = ring_finger.lookup_ring_any(0.5)
 

data/lib/skynet/skynet_active_record_extensions.rb

@@ -11,18 +11,20 @@ class ActiveRecord::Base
     
     jobopts = {
       :single                => true,
-      :map_tasks             => 1,
+      :mappers               => 1,
       :map_data              => [data],
       :name                  => "send_later #{self.class}##{method}",
       :map_name              => "",
-      :map_timeout           => 1.hour,
-      :reduce_timeout        => 1.hour,
-      :master_timeout        => 8.hours,
+      :map_timeout           => 60,
+      :reduce_timeout        => 60,
+      :master_timeout        => 60,
       :master_result_timeout => 1.minute,
-      :map_reduce_class      =>  Skynet::ActiveRecordAsync
+      :map_reduce_class      =>  Skynet::ActiveRecordAsync,
+      :master_retry          => 0,
+      :map_retry             => 0
     }   
     job = Skynet::AsyncJob.new(jobopts)
-    job.run_master
+    job.run
   end
 end
 
@@ -60,7 +62,7 @@ module ActiveRecord
   
   class Mapreduce
     BATCH_SIZE=1000 unless defined?(BATCH_SIZE)
-    
+    MAX_BATCHES_PER_JOB = 1000 unless defined?(MAX_BATCHES_PER_JOB)
     attr_accessor :find_args, :batch_size
     attr_reader :model_class
     
@@ -85,11 +87,17 @@ module ActiveRecord
       end
       new(:find_args => args, :batch_size => args.delete(:batch_size), :model_class => args.delete(:model_class))
     end
+    
+    def log
+      Skynet::Logger.get
+    end
 
     def each_range(opts={})
       opts = opts.clone
       opts[:id] || opts[:id] = 0
       rows = chunk_query(opts)            
+      # log.error "ROWS, #{rows.pretty_print_inspect}"
+      
       
       ii = 0                        
       if rows.empty?
@@ -133,48 +141,60 @@ module ActiveRecord
         # select (@t2:=(@t2+1) % 2) as evenodd, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
 
       mc.connection.execute('select @t1:=0, @t2:=-1, @t3:=0, @t4:=0')
-      mc.connection.select_all("select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.id as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.id #{limit}")
+      sql = "select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.id as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.id #{limit}"
+      # log.error "SQL #{sql}"
+      mc.connection.select_all(sql)
 
       # mc.connection.select_values(mc.send(:construct_finder_sql, :select => "#{mc.table_name}.id", :joins => opts[:joins], :conditions => conditions, :limit => opts[:limit], :order => :id))
     end
     
+    def run_job_for_batch(batches,&block)
+      jobopts = {
+        :mappers               => 20000,
+        :map_data              => batches,
+        :name                  => "each #{model_class} MASTER",
+        :map_name              => "each #{model_class} MAP",
+        :map_timeout           => 60,
+        :master_timeout        => 12.hours,
+        :master_result_timeout => 60,        
+        :master_retry          => 0,
+        :map_retry             => 0
+      }   
+
+      job = nil
+      if block_given?
+        job = Skynet::Job.new(jobopts.merge(:map => block), :local_master => true)
+      else
+        job = Skynet::AsyncJob.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
+      end         
+      job.run
+    end
     
     def each(klass_or_method=nil,&block)    
       klass_or_method ||= model_class
-      batches = []
+      log = Skynet::Logger.get
+
+      batches = []                    
       each_range(find_args) do |ids,ii|
-        batch_count = ids["cnt"].to_i
-        batches[batch_count] = [
-                                  ids['first'].to_i, 
-                                  ids['last'].to_i, 
-                                  find_args.clone,
-                                  model_class
-                                ]
+        batch_item = [
+            ids['first'].to_i, 
+            ids['last'].to_i, 
+            find_args.clone,
+            model_class
+        ]
         if block_given?          
-          batches[batch_count][4] = block
+          batch_item << block
         else
-          batches[batch_count][4] = "#{klass_or_method}" 
+          batch_item << "#{klass_or_method}" 
+        end
+        batches << batch_item
+        if batches.size >= MAX_BATCHES_PER_JOB
+          log.error "MAX BATCH SIZE EXCEEDED RUNNING: #{batches.size}"
+          run_job_for_batch(batches)
+          batches = []    
         end
       end
-      
-      job = nil
-      jobopts = {
-        :map_tasks => 20000,
-        :map_data => batches,
-        :name => "each #{model_class} MASTER",
-        :map_name => "each #{model_class} MAP",
-        :map_timeout      => 1.hour,
-        :reduce_timeout   => 1.hour,
-        :master_timeout => 8.hours,
-        :master_result_timeout => 1.minute
-        
-      }   
-      if block_given?
-        job = Skynet::Job.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
-      else
-        job = Skynet::AsyncJob.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
-      end         
-      job.run
+      run_job_for_batch(batches)      
     end
 
     alias_method :mapreduce, :each
@@ -194,28 +214,40 @@ module ActiveRecord
     end
 
     def self.map(datas)
+      return unless datas and not datas.empty?
       datas.each do |data|    
+        next if (not data.is_a?(Array))
+        next if data.empty?
         model_class = data[3].constantize
         table_name = model_class.table_name
         conditions = "#{table_name}.id >= #{data[0]}"
         conditions += " AND #{table_name}.id <= #{data[1]}" if data[1] > data[0]
         conditions = "(#{conditions})"
         # conditions = "ID BETWEEN #{data[0]} and #{data[1]}"
-        if data[2].empty? or data[2][:conditions].empty?
+        if not data[2]
           data[2] = {:conditions => conditions}
-        else                      
+        elsif data[2].is_a?(Hash) and data[2].empty?
+          data[2] = {:conditions => conditions}
+        elsif data[2].is_a?(Hash) and (not data[2][:conditions] or data[2][:conditions].empty?)
+          data[2][:conditions] = conditions
+        else
           data[2][:conditions] += " AND #{conditions}"
         end                             
         data[2][:select] = "#{table_name}.*"
-        model_class.find(:all, data[2]).each do |ar_object|
+        
+        # log.error "GETTING #{data.pretty_print_inspect}"
+        models = model_class.find(:all, data[2])
+        # log.error "GOT MODELS: #{models.size}"
+        models.each do |ar_object|
           begin
             if data[4].kind_of?(String)
-              begin
-                data[4].constantize.each(ar_object)
-              rescue NameError
-                if ar_object.respond_to?(data[4].to_sym)
-                  ar_object.send(data[4])
-                else
+              if ar_object.respond_to?(data[4].to_sym)
+                # log.error "CALLING #{data[4]} on #{ar_object.class}:#{ar_object.id}"
+                ar_object.send(data[4].to_sym)
+              else
+                begin
+                  data[4].constantize.each(ar_object)
+                rescue NameError
                   raise NameError.new("#{data[4]} is not a class or an instance method in #{model_class}")
                 end
               end

data/lib/skynet/skynet_config.rb

@@ -2,33 +2,60 @@ class Skynet
   LOGDIR = "/var/log"
   
   CONFIG = {
-    :ENABLE                            => true,
-    :SOLO                              => false,
-    :SKYNET_LOG_DIR                    => LOGDIR,
-    :SKYNET_PID_DIR                    => "/tmp",
-    :SKYNET_PIDS_FILE                  => "/tmp/skynet.pid",
-    :SKYNET_LOG_FILE                   => STDOUT,
-    :SKYNET_LOG_LEVEL                  => Logger::ERROR,
-    :SKYNET_LOCAL_MANAGER_URL          => "druby://localhost:40000",
-    :MESSAGE_QUEUE_ADAPTER             => "Skynet::MessageQueueAdapter::TupleSpace",
-    # :TUPLESPACE_DRBURIS              => ["druby://localhost:47647"]
-    # :MESSAGE_QUEUE_ADAPTER           => "Skynet::MessageQueueAdapter::Mysql",
-    # :QUEUE_DATABASE                  => "skynet_queue",
-    # :MYSQL_TEMPERATURE_CHANGE_SLEEP  => 40,
-    :NEXT_TASK_TIMEOUT                 => 60,
-    :USE_RINGSERVER                    => true,
-    :SERVER_HOSTS                      => ["localhost:7647"],
-    :NUMBER_OF_WORKERS                 => 4,
-    :WORKER_CHECK_DELAY                => 40,
-    # :GUID_GENERATOR                  => nil,
-    :PERCENTAGE_OF_TASK_ONLY_WORKERS   => 0.7,
-    :PERCENTAGE_OF_MASTER_ONLY_WORKERS => 0.2
+    :ENABLE                               => true,
+    :SOLO                                 => false,
+    :SKYNET_LOG_DIR                       => LOGDIR,
+    :SKYNET_PID_DIR                       => "/tmp",
+    :SKYNET_PIDS_FILE                     => "/tmp/skynet.pid",
+    :SKYNET_LOG_FILE                      => STDOUT,
+    :SKYNET_LOG_LEVEL                     => Logger::ERROR,
+    :SKYNET_LOCAL_MANAGER_URL             => "druby://localhost:40000",
+    :MESSAGE_QUEUE_ADAPTER                => ("Skynet::MessageQueueAdapter::TupleSpace" || "Skynet::MessageQueueAdapter::Mysql"),
+    :TS_USE_RINGSERVER                    => true,
+    :TS_DRBURIS                           => ["druby://localhost:47647"],   # If you do not use RINGSERVER, you must specifiy the DRBURI
+    :TS_SERVER_HOSTS                      => ["localhost:7647"],
+    :TS_SERVER_START_DELAY                => 10,
+    # :MYSQL_QUEUE_DATABASE                 => "skynet_queue",
+    :MYSQL_TEMPERATURE_CHANGE_SLEEP       => 40,
+    :MYSQL_MESSAGE_QUEUE_TAPLE            => "skynet_message_queues",
+    :MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY => 40,
+    :MYSQL_NEXT_TASK_TIMEOUT              => 60,
+    :MYSQL_ADAPTER                        => "mysql",
+    :MYSQL_HOST                           => "localhost",
+    :MYSQL_DATABASE                       => "skynet",
+    :MYSQL_USERNAME                       => "root",
+    :MYSQL_PASSWORD                       => "",
+    :NUMBER_OF_WORKERS                    => 4,
+    :WORKER_CHECK_DELAY                   => 40,
+    :WORKER_MAX_MEMORY                    => 500,
+    :WORKER_MAX_PROCESSED                 => 1000,
+    :WORKER_VERSION_CHECK_DELAY           => 30,
+    # :GUID_GENERATOR                     => nil,
+    :PERCENTAGE_OF_TASK_ONLY_WORKERS      => 0.7,
+    :PERCENTAGE_OF_MASTER_ONLY_WORKERS    => 0.2,
+    :MAX_RETRIES                          => 6,
+    :DEFAULT_MASTER_RETRY                 => 0,
+    :DEFAULT_MAP_RETRY                    => 3,
+    :DEFAULT_REDUCE_RETRY                 => 3,
+    :DEFAULT_KEEP_MAP_TASKS               => 1,
+    :DEFAULT_KEEP_REDUCE_TASKS            => 1,
+    :MESSAGE_QUEUES                       => ['first', 'second', 'third', 'fourth', 'fifth', 'sixth', 'seventh', 'eighth', 'nineth']
   } unless defined?(CONFIG)
-  
+                           
+  def self.silent
+    if block_given?
+      Skynet.configure(:SKYNET_LOG_LEVEL => 10) do
+        yield
+      end
+    else
+      raise Error.new("Please provide a block to Skynet.silent")
+    end
+  end
   
   def self.configure(config={})
     old_config = CONFIG.dup
     config.each {|k,v| CONFIG[k] = v}
+    Skynet::Logger.log = nil
     if block_given?
       ret = yield
       CONFIG.keys.each do |key|
@@ -56,4 +83,116 @@ class Skynet
     end
     return result
   end
-end
+  
+  # Skynet has many global configuration options.
+  # You can access specific options via Skynet::CONFIG[:OPTION] = ?
+  # You can set many options via                                          
+  #
+  #    Skynet.configure(:OPTION => option, :ANOTHEROPTION => anotheroption)
+  #
+  # If you want specific configuration to only apply to a block of code you can pass configure a block
+  #
+  #    Skynet.configure(:SOMEOPTION => 'value') do
+  #       run code here
+  #    end
+  #
+  # Config Options and current defaults:
+  #  Skynet.configure(
+  #   :ENABLE                               => true,
+  #   :SOLO                                 => false,
+  #   :SKYNET_LOG_DIR                       => LOGDIR,
+  #   :SKYNET_PID_DIR                       => "/tmp",
+  #   :SKYNET_PIDS_FILE                     => "/tmp/skynet.pid",
+  #   :SKYNET_LOG_FILE                      => STDOUT,
+  #   :SKYNET_LOG_LEVEL                     => Logger::ERROR,
+  #   :SKYNET_LOCAL_MANAGER_URL             => "druby://localhost:40000",
+  #   :MESSAGE_QUEUE_ADAPTER                => "Skynet::MessageQueueAdapter::TupleSpace",
+  #   :TS_DRBURIS                           => ["druby://localhost:47647"]
+  #   :TS_USE_RINGSERVER                    => true,
+  #   :TS_SERVER_HOSTS                      => ["localhost:7647"],
+  #   :TS_SERVER_START_DELAY                => 10,
+  #   :MYSQL_QUEUE_DATABASE                 => "skynet_queue",
+  #   :MYSQL_TEMPERATURE_CHANGE_SLEEP       => 40,
+  #   :MYSQL_QUEUE_TEMP_POW                 => 0.6,
+  #   :MYSQL_MESSAGE_QUEUE_TAPLE            => "skynet_message_queues",
+  #   :MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY => 40,
+  #   :MYSQL_NEXT_TASK_TIMEOUT              => 60,
+  #   :WORKER_CHECK_DELAY                   => 40,
+  #   :GUID_GENERATOR                       => nil,
+  #   :NUMBER_OF_WORKERS                    => 4,
+  #   :PERCENTAGE_OF_TASK_ONLY_WORKERS      => 0.7,
+  #   :PERCENTAGE_OF_MASTER_ONLY_WORKERS    => 0.2,
+  #   :MAX_RETRIES                          => 6,
+  #   :DEFAULT_MASTER_RETRY                 => 0,
+  #   :DEFAULT_MAP_RETRY                    => 3,
+  #   :DEFAULT_REDUCE_RETRY                 => 3
+  #   :KEEP_MAP_TASKS                       => false,
+  #   :KEEP_REDUCE_TASKS                    => false
+  #  )
+  #
+  # ENABLE turns Skynet on or off.
+  #
+  # SOLO turns on SOLO mode where Skynet can run in the current process without workers.  
+  # Its almost a Skynet emulation mode
+  #
+  # MESSAGE_QUEUE_ADAPTER
+  # Skynet comes with 2 message queue adaptors 
+  #   Skynet::MessageQueueAdapter::TupleSpace 
+  #   Skynet::MessageQueueAdapter::Mysql
+  # The default is TupleSpace which is an in memory database.
+  # The mysql MQ takes running a migration that comes with skynet_install
+  #
+  # The following only apply to the TupleSpace adapter
+  #   :TS_DRBURIS                           => ["druby://localhost:47647"]
+  #   :TS_USE_RINGSERVER                    => true,
+  #   :TS_SERVER_HOSTS                      => ["localhost:7647"],
+  #
+  # The following only apply to the Mysql adapter
+  #   :MYSQL_QUEUE_DATABASE                 => "skynet_queue",
+  #   :MYSQL_TEMPERATURE_CHANGE_SLEEP       => 40,
+  #   :MYSQL_QUEUE_TEMP_POW                 => 0.6,
+  #   :MYSQL_MESSAGE_QUEUE_TAPLE            => "skynet_message_queues",
+  #   :MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY => 40,
+  #   :MYSQL_NEXT_TASK_TIMEOUT              => 60,
+
+  class Config     
+    include Enumerable
+
+    def each
+      Skynet::CONFIG.each {|k,v| yield k,v}
+    end
+
+    def add_message_queue(queue_name)
+      self.message_queues << queue_name
+    end
+    
+    def queue_id_by_name(queue_name)
+      if Skynet::CONFIG[:MESSAGE_QUEUES].index(queue_name)
+        return Skynet::CONFIG[:MESSAGE_QUEUES].index(queue_name)
+      else     
+        raise Skynet::Error("#{queue_name} is not a valid queue")
+      end
+    end       
+    
+    def queue_name_by_id(queue_id)               
+      queue_id = queue_id.to_i
+      if Skynet::CONFIG[:MESSAGE_QUEUES][queue_id]
+        return Skynet::CONFIG[:MESSAGE_QUEUES][queue_id]
+      else     
+        raise Skynet::Error("#{queue_id} is not a valid queue_id")
+      end
+    end
+    
+    
+    
+    def method_missing(name, *args)
+      name = name.to_s.upcase.to_sym
+      if name.to_s =~ /^(.*)=$/
+        name = $1.to_sym
+        Skynet::CONFIG[name] = args.first
+      else
+        Skynet::CONFIG[name]
+      end
+    end
+  end
+end