skynet 0.9.2 → 0.9.3

data/lib/skynet/message_queue_adapters/tuple_space.rb

@@ -10,17 +10,17 @@ end
 class Skynet
   class Error < StandardError
   end
-  
+
   class RequestExpiredError < Skynet::Error
 	end
-	
+
 	class InvalidMessage < Skynet::Error
   end
 
   class MessageQueueAdapter
-  	
+
     class TupleSpace < Skynet::MessageQueueAdapter
-      
+
       include SkynetDebugger
 
       USE_FALLBACK_TASKS = true
@@ -36,11 +36,11 @@ class Skynet
         :tuplespace
       end
 
-      def initialize(ts=nil)
-        if not ts
-          ts = self.class.get_tuple_space
-        end
-        @ts = ts
+      attr_accessor :start_options
+      
+      def initialize(options={})
+        @start_options = options
+        @ts = self.class.get_tuple_space(options)
       end
 
       def take_next_task(curver,timeout=nil,payload_type=nil,queue_id=0)
@@ -72,36 +72,6 @@ class Skynet
         take_fallback_message(message)
       end
 
-
-      def write_worker_status(task, timeout=nil)
-        begin
-          take_worker_status(task,0.00001)
-        rescue Skynet::RequestExpiredError
-        end   
-        write(Skynet::WorkerStatusMessage.new(task), timeout)
-      end
-
-      def take_worker_status(task, timeout=nil)
-        Skynet::WorkerStatusMessage.new(take(Skynet::WorkerStatusMessage.worker_status_template(task), timeout))
-      end
-
-      def read_all_worker_statuses(hostname=nil)
-        ws = Skynet::WorkerStatusMessage.all_workers_template(hostname)
-        workers = read_all(ws).collect{ |w| Skynet::WorkerStatusMessage.new(w) }#.sort{ |a,b| a.process_id <=> b.process_id }
-      end
-
-      def clear_worker_status(hostname=nil)
-        cnt = 0
-        begin
-          loop do
-            take(Skynet::WorkerStatusMessage.new([:status, :worker, hostname, nil, nil]),0.01)
-            cnt += 1
-          end
-        rescue Skynet::RequestExpiredError
-        end
-        cnt
-      end
-
       def list_tasks(iteration=nil,queue_id=0)
         read_all(Skynet::Message.outstanding_tasks_template(iteration,queue_id))
       end
@@ -165,20 +135,20 @@ class Skynet
         rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
           error "ERROR #{e.inspect}", caller
         end
-    
+
         tasks.size.times do |ii|
           take(Skynet::Message.outstanding_tasks_template,0.00001)
         end
-    
+
         results = read_all(Skynet::Message.outstanding_results_template)
         results.size.times do |ii|
           take(Skynet::Message.outstanding_results_template,0.00001)
         end
-    
+
         task_tuples = read_all(Skynet::Message.outstanding_tasks_template)
         result_tuples = read_all(Skynet::Message.outstanding_results_template)
         return task_tuples + result_tuples
-      end  
+      end
 
       def stats
         t1 = Time.now
@@ -188,7 +158,7 @@ class Skynet
         p_tasks = tasks.partition {|task| task[9] == 0}
         {:taken_tasks => p_tasks[1].size, :untaken_tasks => p_tasks[0].size, :results => list_results.size, :time => t2.to_f}
       end
-      
+
       private
 
       attr_accessor :ts
@@ -208,7 +178,7 @@ class Skynet
       def read_all(template)
         ts_command(:read_all,template)
       end
-      
+
       ###### FALLBACK METHODS
       def write_fallback_task(message)
         return unless USE_FALLBACK_TASKS
@@ -243,25 +213,25 @@ class Skynet
         else
           raise InvalidMessage.new("You must provide a valid Skynet::Message object when calling #{command}.  You passed #{message.inspect}.")
         end
-    
+
         begin
           if command==:read_all
             return ts.send(command,tuple)
           else
             return ts.send(command,tuple,timeout)
           end
-    
+
         rescue Rinda::RequestExpiredError
           raise Skynet::RequestExpiredError.new
         rescue DRb::DRbConnError => e
           begin
-            log.error "Couldnt run command [#{command}] on tuplespace."
-            @ts = self.class.get_tuple_space
-            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect}")
+            error "Couldnt run command [#{command}] on tuplespace. start options: #{@start_options.inspect}"
+            @ts = self.class.get_tuple_space(@start_options)
+            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect} #{@start_options.inspect}")
             # tries += 1
             # next
           rescue Skynet::ConnectionError => e
-            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect}")
+            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect} #{@start_options.inspect}")
           # rescue RuntimeError => e
           #   raise Skynet::ConnectionError.new("Can't find ring finger. #{}")
           rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
@@ -276,42 +246,46 @@ class Skynet
   ####################################
 
       ### XXX ACCEPT MULTIPLE TUPLE SPACES and a flag whether to use replication or failover.
-    
-      def self.get_tuple_space
-        return @@ts if is_valid_tuplespace?(@@ts)
+
+      def self.get_tuple_space(options = {})    
+        use_ringserver   = options[:use_ringserver]
+        ringserver_hosts = options[:ringserver_hosts]
+        drburi           = options[:drburi]
+        
+        return @@ts if valid_tuplespace?(@@ts)
         loop do
           begin
             DRb.start_service
-            if Skynet::CONFIG[:TS_USE_RINGSERVER]
-              Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx] =~ /(.+):(\d+)/
+            if use_ringserver
+              ringserver_hosts[@@curhostidx] =~ /(.+):(\d+)/
               host = $1
               port = $2.to_i
-              @@ts = connect_to_tuple_space(host,port)
+              @@ts = connect_to_tuple_space(host,port,use_ringserver)
             else
-              drburi = Skynet::CONFIG[:TS_DRBURIS].first
-              drburi = "druby://#{drburi}" unless drburi =~ %r{druby://}              
+              drburi = "druby://#{drburi}" unless drburi =~ %r{druby://}
               @@ts = get_tuple_space_from_drburi(drburi)
-              log.info "#{self} CONNECTED TO #{drburi}"
+              raise DRb::DRbConnError.new unless valid_tuplespace?(@@ts)
+              info "#{self} CONNECTED TO #{drburi}"
             end
             return @@ts
           rescue RuntimeError => e
-            if Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx + 1]
-              log.error "#{self} Couldn't connect to #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx]} trying #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx+1]}"
+            if ringserver_hosts[@@curhostidx + 1]
+              error "#{self} Couldn't connect to #{ringserver_hosts[@@curhostidx]} trying #{ringserver_hosts[@@curhostidx+1]}"
               @@curhostidx += 1
               next
             else
-              raise Skynet::ConnectionError.new("Can't find ring finger @ #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")
+              raise Skynet::ConnectionError.new("Can't find ring finger @ #{ringserver_hosts[@@curhostidx]}. #{e.class} #{e.message}")
             end
           rescue Exception => e
-            raise Skynet::ConnectionError.new("Error getting tuplespace @ #{Skynet::CONFIG[:TS_SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")          
+            raise Skynet::ConnectionError.new("Error getting tuplespace @ #{ringserver_hosts[@@curhostidx]}. #{e.class} #{e.message}")
           end
         end
         return @@ts
       end
-    
-      def self.connect_to_tuple_space(host,port)
-        log.info "#{self} trying to connect to #{host}:#{port}"
-        if Skynet::CONFIG[:TS_USE_RINGSERVER]
+
+      def self.connect_to_tuple_space(host,port,use_ringserver=Skynet::CONFIG[:TS_USE_RINGSERVER])
+        info "#{self} trying to connect to #{host}:#{port}"
+        if use_ringserver
           ring_finger = Rinda::RingFinger.new(host,port)
           ring_server = ring_finger.lookup_ring_any(0.5)
 
@@ -320,15 +294,15 @@ class Skynet
         else
           ts = get_tuple_space_from_drburi("druby://#{host}:#{port}")
         end
-        log.info "#{self} CONNECTED TO #{host}:#{port}"
+        info "#{self} CONNECTED TO #{host}:#{port}"
         ts
       end
-      
+
       def self.get_tuple_space_from_drburi(drburi)
         DRbObject.new(nil, drburi)
       end
 
-      def self.is_valid_tuplespace?(ts)
+      def self.valid_tuplespace?(ts)
         return false unless ts
         begin
           ts.read_all([:valid])

data/lib/skynet/skynet_active_record_extensions.rb

@@ -66,16 +66,23 @@ module ActiveRecord
     attr_accessor :find_args, :batch_size
     attr_reader :model_class
     
+    delegate :primary_key, :table_name, :to => :model_klass
+    delegate :execute, :select_all, :to => 'model_klass.connection'
+    
     def initialize(options = {})
       @find_args = options[:find_args]
       @batch_size = options[:batch_size] || BATCH_SIZE
       @model_class = options[:model_class]
     end
 
+    def model_klass
+      @model_klass ||= model_class.constantize
+    end
+
     def model_class=(model_c)
       @model_class = model_c.to_s
     end
-    
+
     def self.find(*args)
       if not args.first.is_a?(Hash)
         args.shift
@@ -95,6 +102,11 @@ module ActiveRecord
     def each_range(opts={})
       opts = opts.clone
       opts[:id] || opts[:id] = 0
+      count = model_klass.count(:all,:conditions => opts[:conditions], :joins => opts[:joins])
+      if count <= batch_size
+        return yield({"first" => 0, "last" => nil, "cnt" => 0}, 0)
+      end
+      
       rows = chunk_query(opts)            
       # log.error "ROWS, #{rows.pretty_print_inspect}"
       
@@ -121,10 +133,8 @@ module ActiveRecord
     end  
 
     def chunk_query(opts={})
-      mc = model_class.constantize
-      table_name = mc.table_name
 
-      conditions = "#{table_name}.id > #{opts[:id]} AND ((@t1:=(@t1+1) % #{batch_size})=0)"
+      conditions = "#{table_name}.#{primary_key} > #{opts[:id]} AND ((@t1:=(@t1+1) % #{batch_size})=0)"
       opts = opts.clone                                                               
       if opts[:conditions].nil? or opts[:conditions].empty?
         opts[:conditions] = conditions
@@ -140,10 +150,10 @@ module ActiveRecord
         # select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
         # select (@t2:=(@t2+1) % 2) as evenodd, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
 
-      mc.connection.execute('select @t1:=0, @t2:=-1, @t3:=0, @t4:=0')
-      sql = "select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.id as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.id #{limit}"
+      execute('select @t1:=0, @t2:=-1, @t3:=0, @t4:=0')
+      sql = "select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.#{primary_key} as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.#{primary_key} #{limit}"
       # log.error "SQL #{sql}"
-      mc.connection.select_all(sql)
+      select_all(sql)
 
       # mc.connection.select_values(mc.send(:construct_finder_sql, :select => "#{mc.table_name}.id", :joins => opts[:joins], :conditions => conditions, :limit => opts[:limit], :order => :id))
     end
@@ -170,7 +180,7 @@ module ActiveRecord
       job.run
     end
     
-    def each(klass_or_method=nil,&block)    
+    def map(klass_or_method=nil,&block)    
       klass_or_method ||= model_class
       log = Skynet::Logger.get
 
@@ -197,7 +207,8 @@ module ActiveRecord
       run_job_for_batch(batches)      
     end
 
-    alias_method :mapreduce, :each
+    alias_method :each, :map
+    alias_method :mapreduce, :map
 
     def model_class
       @model_class || self.class.model_class
@@ -220,8 +231,8 @@ module ActiveRecord
         next if data.empty?
         model_class = data[3].constantize
         table_name = model_class.table_name
-        conditions = "#{table_name}.id >= #{data[0]}"
-        conditions += " AND #{table_name}.id <= #{data[1]}" if data[1] > data[0]
+        conditions = "#{table_name}.#{model_class.primary_key} >= #{data[0]}"
+        conditions += " AND #{table_name}.#{model_class.primary_key} <= #{data[1]}" if data[1] > data[0]
         conditions = "(#{conditions})"
         # conditions = "ID BETWEEN #{data[0]} and #{data[1]}"
         if not data[2]

data/lib/skynet/skynet_config.rb

@@ -1,18 +1,19 @@
 class Skynet       
-  LOGDIR = "/var/log"
-  
+
   CONFIG = {
     :ENABLE                               => true,
     :SOLO                                 => false,
-    :SKYNET_LOG_DIR                       => LOGDIR,
-    :SKYNET_PID_DIR                       => "/tmp",
-    :SKYNET_PIDS_FILE                     => "/tmp/skynet.pid",
-    :SKYNET_LOG_FILE                      => STDOUT,
+    :APP_ROOT                             => nil,
+    :SKYNET_LOG_DIR                       => nil,
+    :SKYNET_PID_DIR                       => nil,
+    :SKYNET_PID_FILE                      => "skynet.pid",
+    :SKYNET_LOG_FILE                      => "skynet.log",
+    :SKYNET_MANAGER_STATS_FILE            => "skynet_manager_stats.txt",
     :SKYNET_LOG_LEVEL                     => Logger::ERROR,
-    :SKYNET_LOCAL_MANAGER_URL             => "druby://localhost:40000",
+    :SKYNET_LOCAL_MANAGER_PORT            => "40000",
     :MESSAGE_QUEUE_ADAPTER                => ("Skynet::MessageQueueAdapter::TupleSpace" || "Skynet::MessageQueueAdapter::Mysql"),
     :TS_USE_RINGSERVER                    => true,
-    :TS_DRBURIS                           => ["druby://localhost:47647"],   # If you do not use RINGSERVER, you must specifiy the DRBURI
+    :TS_DRBURI                            => "druby://localhost:47647",   # If you do not use RINGSERVER, you must specifiy the DRBURI
     :TS_SERVER_HOSTS                      => ["localhost:7647"],
     :TS_SERVER_START_DELAY                => 10,
     # :MYSQL_QUEUE_DATABASE                 => "skynet_queue",
@@ -22,11 +23,11 @@ class Skynet
     :MYSQL_NEXT_TASK_TIMEOUT              => 60,
     :MYSQL_ADAPTER                        => "mysql",
     :MYSQL_HOST                           => "localhost",
-    :MYSQL_DATABASE                       => "skynet",
-    :MYSQL_USERNAME                       => "root",
+    :MYSQL_DATABASE                       => nil, # 'skynet'
+    :MYSQL_USERNAME                       => nil,
     :MYSQL_PASSWORD                       => "",
     :NUMBER_OF_WORKERS                    => 4,
-    :WORKER_CHECK_DELAY                   => 40,
+    :WORKER_CHECK_DELAY                   => 5,
     :WORKER_MAX_MEMORY                    => 500,
     :WORKER_MAX_PROCESSED                 => 1000,
     :WORKER_VERSION_CHECK_DELAY           => 30,
@@ -65,6 +66,10 @@ class Skynet
       ret
     end
   end
+  
+  def self.config
+    Skynet::Config.new
+  end
 
   def self.solo(config = {}) 
     raise Skynet::Error.new("You provide a code block to Skynet.solo") unless block_given?
@@ -100,14 +105,14 @@ class Skynet
   #  Skynet.configure(
   #   :ENABLE                               => true,
   #   :SOLO                                 => false,
-  #   :SKYNET_LOG_DIR                       => LOGDIR,
   #   :SKYNET_PID_DIR                       => "/tmp",
-  #   :SKYNET_PIDS_FILE                     => "/tmp/skynet.pid",
-  #   :SKYNET_LOG_FILE                      => STDOUT,
+  #   :SKYNET_PID_FILE                      => "skynet.pid",
+  #   :SKYNET_LOG_DIR                       => ENV["HOME"],
+  #   :SKYNET_LOG_FILE                      => "skynet.log",
   #   :SKYNET_LOG_LEVEL                     => Logger::ERROR,
   #   :SKYNET_LOCAL_MANAGER_URL             => "druby://localhost:40000",
   #   :MESSAGE_QUEUE_ADAPTER                => "Skynet::MessageQueueAdapter::TupleSpace",
-  #   :TS_DRBURIS                           => ["druby://localhost:47647"]
+  #   :TS_DRBURI                            => "druby://localhost:47647"
   #   :TS_USE_RINGSERVER                    => true,
   #   :TS_SERVER_HOSTS                      => ["localhost:7647"],
   #   :TS_SERVER_START_DELAY                => 10,
@@ -143,7 +148,7 @@ class Skynet
   # The mysql MQ takes running a migration that comes with skynet_install
   #
   # The following only apply to the TupleSpace adapter
-  #   :TS_DRBURIS                           => ["druby://localhost:47647"]
+  #   :TS_DRBURI                            => "druby://localhost:47647"
   #   :TS_USE_RINGSERVER                    => true,
   #   :TS_SERVER_HOSTS                      => ["localhost:7647"],
   #
@@ -162,11 +167,11 @@ class Skynet
       Skynet::CONFIG.each {|k,v| yield k,v}
     end
 
-    def add_message_queue(queue_name)
+    def self.add_message_queue(queue_name)
       self.message_queues << queue_name
     end
     
-    def queue_id_by_name(queue_name)
+    def self.queue_id_by_name(queue_name)
       if Skynet::CONFIG[:MESSAGE_QUEUES].index(queue_name)
         return Skynet::CONFIG[:MESSAGE_QUEUES].index(queue_name)
       else     
@@ -174,7 +179,7 @@ class Skynet
       end
     end       
     
-    def queue_name_by_id(queue_id)               
+    def self.queue_name_by_id(queue_id)               
       queue_id = queue_id.to_i
       if Skynet::CONFIG[:MESSAGE_QUEUES][queue_id]
         return Skynet::CONFIG[:MESSAGE_QUEUES][queue_id]
@@ -183,9 +188,38 @@ class Skynet
       end
     end
     
+    def self.logfile_location
+      if skynet_log_file.is_a?(String) and skynet_log_dir
+        skynet_log_dir.sub(/\/$/,'') + "/" + skynet_log_file.sub(/^\//,'')
+      else
+        skynet_log_file
+      end
+    end
+
+    def self.pidfile_location
+      if skynet_pid_dir and skynet_pid_file
+        skynet_pid_dir.sub(/\/$/,'') + "/" + skynet_pid_file.sub(/^\//,'')
+      else
+        skynet_pid_dir
+      end
+    end
     
+    def manager_statfile_location
+      if skynet_log_dir.is_a?(String) and skynet_log_dir
+        skynet_log_dir.sub(/\/$/,'') + "/" + skynet_manager_stats_file.sub(/^\//,'')
+      else
+        skynet_log_dir
+      end
+    end    
+    def method_missing(name,*args)
+      if self.class.respond_to?(name)
+        self.class.send(name,*args)
+      else        
+        self.class.method_missing(name,*args)
+      end
+    end                               
     
-    def method_missing(name, *args)
+    def self.method_missing(name, *args)
       name = name.to_s.upcase.to_sym
       if name.to_s =~ /^(.*)=$/
         name = $1.to_sym