skynet 0.9.1

data/lib/skynet/message_queue_adapters/tuple_space.rb

@@ -0,0 +1,327 @@
+require 'rinda/tuplespace'
+class Rinda::TupleSpaceProxy
+  def take(tuple, sec=nil, &block)
+    port = []
+    port.push @ts.move(nil, tuple, sec, &block)
+    port[0]
+  end
+end
+
+class Skynet
+  class Error < StandardError
+  end
+  
+  class RequestExpiredError < Skynet::Error
+	end
+	
+	class InvalidMessage < Skynet::Error
+  end
+
+  class MessageQueueAdapter
+  	
+    class TupleSpace < Skynet::MessageQueueAdapter
+      
+      include SkynetDebugger
+
+      USE_FALLBACK_TASKS = true
+
+      @@ts = nil
+      @@curhostidx = 0
+
+      def self.debug_class_desc
+        "TUPLESPACE"
+      end
+
+      def self.adapter
+        :tuplespace
+      end
+
+      def initialize(ts=nil)
+        if not ts
+          ts = self.class.get_tuple_space
+        end
+        @ts = ts
+      end
+
+      def take_next_task(curver,timeout=nil,payload_type=nil)
+        message = Skynet::Message.new(take(Skynet::Message.next_task_template(curver,payload_type),timeout))
+        write_fallback_task(message)
+        message
+      end
+
+      def write_message(message,timeout=nil)
+        write(message,timeout)
+      end
+
+      def write_result(message,result=[],timeout=nil)
+        result_message = message.result_message(result).to_a
+        write(result_message,timeout)
+        take_fallback_message(message)
+        result_message
+      end
+
+      def take_result(job_id,timeout=nil)
+        Skynet::Message.new(take(Skynet::Message.result_template(job_id),timeout))
+      end
+
+      def write_error(message,error='',timeout=nil)
+        write(message.error_message(error),timeout)
+        take_fallback_message(message)
+      end
+
+
+      def write_worker_status(task, timeout=nil)
+        begin
+          take_worker_status(task,0.00001)
+        rescue Skynet::RequestExpiredError
+        end   
+        write(Skynet::WorkerStatusMessage.new(task), timeout)
+      end
+
+      def take_worker_status(task, timeout=nil)
+        Skynet::WorkerStatusMessage.new(take(Skynet::WorkerStatusMessage.worker_status_template(task), timeout))
+      end
+
+      def read_all_worker_statuses(hostname=nil)
+        ws = Skynet::WorkerStatusMessage.all_workers_template(hostname)
+        workers = read_all(ws).collect{ |w| Skynet::WorkerStatusMessage.new(w) }#.sort{ |a,b| a.process_id <=> b.process_id }
+      end
+
+      def clear_worker_status(hostname=nil)
+        cnt = 0
+        begin
+          loop do
+            take(Skynet::WorkerStatusMessage.new([:status, :worker, hostname, nil, nil]),0.01)
+            cnt += 1
+          end
+        rescue Skynet::RequestExpiredError
+        end
+        cnt
+      end
+
+      def list_tasks(iteration=nil)
+        read_all(Skynet::Message.outstanding_tasks_template(iteration))
+      end
+
+      def list_results
+        read_all(Skynet::Message.outstanding_results_template)
+      end
+
+      def get_worker_version
+        begin
+          message = Skynet::WorkerVersionMessage.new(read(Skynet::WorkerVersionMessage.template,0.00001))
+          if message
+            curver = message.version
+          else
+            curver=0
+          end
+        rescue Skynet::RequestExpiredError => e
+          curver = 0
+        end
+        curver
+      end
+
+      def set_worker_version(ver=nil)
+        begin
+          messages = read_all(Skynet::WorkerVersionMessage.template).collect {|ret| Skynet::WorkerVersionMessage.new(ret)}
+          curver = 0
+          messages.each do |message|
+            curver = message.version
+            debug "CURRENT WORKER VERSION #{curver}"
+            curvmessage = Skynet::WorkerVersionMessage.new(take(message.template,0.00001))
+            if curvmessage
+              curver = curvmessage.version
+            else
+              curver=0
+            end
+          end
+        rescue Skynet::RequestExpiredError => e
+          curver = 0
+        end
+
+        newver = ver ? ver : curver + 1
+        debug "WRITING CURRENT WORKER REV #{newver}"
+        write(Skynet::WorkerVersionMessage.new(:version=>newver))
+        newver
+      end
+
+      def clear_outstanding_tasks
+        begin
+          tasks = read_all(Skynet::Message.outstanding_tasks_template)
+        rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
+          error "ERROR #{e.inspect}", caller
+        end
+    
+        tasks.size.times do |ii|
+          take(Skynet::Message.outstanding_tasks_template,0.00001)
+        end
+    
+        results = read_all(Skynet::Message.outstanding_results_template)
+        results.size.times do |ii|
+          take(Skynet::Message.outstanding_results_template,0.00001)
+        end
+    
+        task_tuples = read_all(Skynet::Message.outstanding_tasks_template)
+        result_tuples = read_all(Skynet::Message.outstanding_results_template)
+        return task_tuples + result_tuples
+      end  
+
+      def stats
+        t1 = Time.now
+        tasks = list_tasks
+        results = list_results
+        t2 = Time.now - t1
+        p_tasks = tasks.partition {|task| task[9] == 0}
+        {:taken_tasks => p_tasks[1].size, :untaken_tasks => p_tasks[0].size, :results => list_results.size, :time => t2.to_f}
+      end
+      
+      private
+
+      attr_accessor :ts
+
+      def write(tuple,timeout=nil)
+        ts_command(:write,tuple,timeout)
+      end
+
+      def take(template,timeout=nil)
+        ts_command(:take,template,timeout)
+      end
+
+      def read(template,timeout=nil)
+        ts_command(:read,template,timeout)
+      end
+
+      def read_all(template)
+        ts_command(:read_all,template)
+      end
+      
+      ###### FALLBACK METHODS
+      def write_fallback_task(message)
+        return unless USE_FALLBACK_TASKS
+        debug "4 WRITING BACKUP TASK #{message.task_id}", @fallback_worker_message
+        ftm = message.fallback_task_message
+        debug "WRITE FALLBACK TASK", ftm.to_a
+        timeout = message.expiry * 8
+        write(ftm,timeout)
+        ftm
+      end
+
+      def take_fallback_message(message,timeout=0.01)
+        return unless USE_FALLBACK_TASKS
+        begin
+          # debug "LOOKING FOR FALLBACK TEMPLATE", message.fallback_template
+          fb_message = Skynet::Message.new(take(message.fallback_template,timeout))
+          # debug "TOOK FALLBACK MESSAGE for TASKID: #{fb_message.task_id}"
+        rescue Skynet::RequestExpiredError => e
+          error "Couldn't find expected FALLBACK MESSAGE"
+        end
+      end
+      ## END FALLBACK METHODS
+
+      def ts_command(command,message,timeout=nil)
+        # tries = 0
+        # until(tries > 3)
+        if message.is_a?(Skynet::Message)
+          tuple = message.to_a
+        elsif message.is_a?(Array)
+          tuple = message
+        else
+          raise InvalidMessage.new("You must provide a valid Skynet::Message object when calling #{command}.  You passed #{message.inspect}.")
+        end
+    
+        begin
+          if command==:read_all
+            return ts.send(command,tuple)
+          else
+            return ts.send(command,tuple,timeout)
+          end
+    
+        rescue Rinda::RequestExpiredError
+          raise Skynet::RequestExpiredError.new
+        rescue DRb::DRbConnError => e
+          begin
+            log.error "Couldnt run command [#{command}] on tuplespace."
+            @ts = self.class.get_tuple_space
+            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect}")
+            # tries += 1
+            # next
+          rescue Skynet::ConnectionError => e
+            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect}")
+          # rescue RuntimeError => e
+          #   raise Skynet::ConnectionError.new("Can't find ring finger. #{}")
+          rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
+            raise Skynet::ConnectionError.new("There was a problem conected to the #{self.class} #{e.class} #{e.message}")
+          end
+        end
+        # end
+      end
+
+  ####################################
+  ######## CLASS METHODS #############
+  ####################################
+
+      ### XXX ACCEPT MULTIPLE TUPLE SPACES and a flag whether to use replication or failover.
+    
+      def self.get_tuple_space
+        return @@ts if is_valid_tuplespace?(@@ts)
+        loop do
+          begin
+            DRb.start_service
+            if Skynet::CONFIG[:USE_RINGSERVER]
+              Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx] =~ /(.+):(\d+)/
+              host = $1
+              port = $2.to_i
+              @@ts = connect_to_tuple_space(host,port)
+            else
+              drburi = Skynet::CONFIG[:TUPLESPACE_DRBURIS].first
+              drburi = "druby://#{drburi}" unless drburi =~ %r{druby://}              
+              @@ts = get_tuple_space_from_drburi(drburi)
+              log.info "#{self} CONNECTED TO #{drburi}"
+            end
+            return @@ts
+          rescue RuntimeError => e
+            if Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx + 1]
+              log.error "#{self} Couldn't connect to #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]} trying #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx+1]}"
+              @@curhostidx += 1
+              next
+            else
+              raise Skynet::ConnectionError.new("Can't find ring finger @ #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")
+            end
+          rescue Exception => e
+            raise Skynet::ConnectionError.new("Error getting tuplespace @ #{Skynet::CONFIG[:SERVER_HOSTS][@@curhostidx]}. #{e.class} #{e.message}")          
+          end
+        end
+        return @@ts
+      end
+    
+      def self.connect_to_tuple_space(host,port)
+        log.info "#{self} trying to connect to #{host}:#{port}"
+        if Skynet::CONFIG[:USE_RINGSERVER]
+          ring_finger = Rinda::RingFinger.new(host,port)
+          ring_server = ring_finger.lookup_ring_any(0.5)
+
+          ringts = ring_server.read([:name, :TupleSpace, nil, nil],0.00005)[2]
+          ts = Rinda::TupleSpaceProxy.new(ringts)
+        else
+          ts = get_tuple_space_from_drburi("druby://#{host}:#{port}")
+        end
+        log.info "#{self} CONNECTED TO #{host}:#{port}"
+        ts
+      end
+      
+      def self.get_tuple_space_from_drburi(drburi)
+        DRbObject.new(nil, drburi)
+      end
+
+      def self.is_valid_tuplespace?(ts)
+        return false unless ts
+        begin
+          ts.read_all([:valid])
+          return true
+        rescue DRb::DRbConnError, RuntimeError, Errno::ECONNREFUSED  => e
+          return false
+        end
+      end
+    end
+  end
+end

data/lib/skynet/skynet_active_record_extensions.rb

@@ -0,0 +1,237 @@
+class ActiveRecord::Base
+  def send_later(method,opts=nil,save=nil)
+    raise NoMethodError.new("Method: #{method} doesn't exist in #{self.class}") unless self.respond_to?(method)
+    data = { 
+      :model_class => self.class.to_s, 
+      :model_id => self.id, 
+      :method => method,       
+    }
+    data[:save] = 1 if save
+    data[:opts] = opts.to_yaml if opts
+    
+    jobopts = {
+      :single                => true,
+      :map_tasks             => 1,
+      :map_data              => [data],
+      :name                  => "send_later #{self.class}##{method}",
+      :map_name              => "",
+      :map_timeout           => 1.hour,
+      :reduce_timeout        => 1.hour,
+      :master_timeout        => 8.hours,
+      :master_result_timeout => 1.minute,
+      :map_reduce_class      =>  Skynet::ActiveRecordAsync
+    }   
+    job = Skynet::AsyncJob.new(jobopts)
+    job.run_master
+  end
+end
+
+class <<ActiveRecord::Base
+  def distributed_find(*args)
+    all = ActiveRecord::Mapreduce.find(*args)
+    all.model_class = self
+    all
+  end
+end  
+
+class Skynet::ActiveRecordAsync
+  include SkynetDebugger
+  
+  def self.map(datas)
+    datas.each do |data| 
+      begin            
+        model = data[:model_class].constantize.find(data[:model_id])
+        if data[:opts]
+          model.send(data[:method].to_sym, YAML.load(data[:opts]))
+        else
+          model.send(data[:method].to_sym)
+        end
+        model.save if data[:save]
+      rescue Exception => e
+        error "Error in #{self} #{e.inspect}"
+      end
+    end
+    return
+  end
+end
+
+module ActiveRecord  
+  
+  
+  class Mapreduce
+    BATCH_SIZE=1000 unless defined?(BATCH_SIZE)
+    
+    attr_accessor :find_args, :batch_size
+    attr_reader :model_class
+    
+    def initialize(options = {})
+      @find_args = options[:find_args]
+      @batch_size = options[:batch_size] || BATCH_SIZE
+      @model_class = options[:model_class]
+    end
+
+    def model_class=(model_c)
+      @model_class = model_c.to_s
+    end
+    
+    def self.find(*args)
+      if not args.first.is_a?(Hash)
+        args.shift
+      end       
+      if args.nil? or args.empty?
+        args = {} 
+      else
+        args = *args
+      end
+      new(:find_args => args, :batch_size => args.delete(:batch_size), :model_class => args.delete(:model_class))
+    end
+
+    def each_range(opts={})
+      opts = opts.clone
+      opts[:id] || opts[:id] = 0
+      rows = chunk_query(opts)            
+      
+      ii = 0                        
+      if rows.empty?
+        rows = [{"first" => 0, "last" => nil, "cnt" => ii}]
+      end
+      last_row = nil
+      while rows.any?
+        rows.each do |record| 
+          last_row = record
+          yield record, ii          
+        end
+        ii +=1                                                  
+        return if last_row["last"].nil?
+        rows = chunk_query(opts.merge(:id => rows.last["last"])) 
+      end
+      
+      if last_row["last"] and (last_row["last"].to_i - last_row["first"].to_i) >= batch_size 
+        catchall_row = {"first" => last_row["last"].to_i+1, "last" => nil, "cnt" => ii}
+        yield catchall_row, ii
+      end
+    end  
+
+    def chunk_query(opts={})
+      mc = model_class.constantize
+      table_name = mc.table_name
+
+      conditions = "#{table_name}.id > #{opts[:id]} AND ((@t1:=(@t1+1) % #{batch_size})=0)"
+      opts = opts.clone                                                               
+      if opts[:conditions].nil? or opts[:conditions].empty?
+        opts[:conditions] = conditions
+      else
+        opts[:conditions] += " AND " unless opts[:conditions].empty?
+        opts[:conditions] += conditions
+      end                
+      limit = opts[:limit] ? "LIMIT #{opts[:limit]}" : nil
+      # select @t2:=(@t2+1), @t3:=@t4, @t4:=id from profiles where ( ((@t1:=(@t1+1) % 1000)=0) or (((@t1+1) % 1000)=0) )  order by id LIMIT 100;
+
+      # BEST
+      # select @t1:=0, @t2:=0, @t3:=0, @t4:=0;
+        # select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
+        # select (@t2:=(@t2+1) % 2) as evenodd, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
+
+      mc.connection.execute('select @t1:=0, @t2:=-1, @t3:=0, @t4:=0')
+      mc.connection.select_all("select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.id as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.id #{limit}")
+
+      # mc.connection.select_values(mc.send(:construct_finder_sql, :select => "#{mc.table_name}.id", :joins => opts[:joins], :conditions => conditions, :limit => opts[:limit], :order => :id))
+    end
+    
+    
+    def each(klass_or_method=nil,&block)    
+      klass_or_method ||= model_class
+      batches = []
+      each_range(find_args) do |ids,ii|
+        batch_count = ids["cnt"].to_i
+        batches[batch_count] = [
+                                  ids['first'].to_i, 
+                                  ids['last'].to_i, 
+                                  find_args.clone,
+                                  model_class
+                                ]
+        if block_given?          
+          batches[batch_count][4] = block
+        else
+          batches[batch_count][4] = "#{klass_or_method}" 
+        end
+      end
+      
+      job = nil
+      jobopts = {
+        :map_tasks => 20000,
+        :map_data => batches,
+        :name => "each #{model_class} MASTER",
+        :map_name => "each #{model_class} MAP",
+        :map_timeout      => 1.hour,
+        :reduce_timeout   => 1.hour,
+        :master_timeout => 8.hours,
+        :master_result_timeout => 1.minute
+        
+      }   
+      if block_given?
+        job = Skynet::Job.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
+      else
+        job = Skynet::AsyncJob.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
+      end         
+      job.run
+    end
+
+    alias_method :mapreduce, :each
+
+    def model_class
+      @model_class || self.class.model_class
+    end
+    
+    def self.model_class(model_class)
+      (class << self; self; end).module_eval do
+         define_method(:model_class) {model_class}
+      end      
+    end
+    
+    def self.log
+      Skynet::Logger.get
+    end
+
+    def self.map(datas)
+      datas.each do |data|    
+        model_class = data[3].constantize
+        table_name = model_class.table_name
+        conditions = "#{table_name}.id >= #{data[0]}"
+        conditions += " AND #{table_name}.id <= #{data[1]}" if data[1] > data[0]
+        conditions = "(#{conditions})"
+        # conditions = "ID BETWEEN #{data[0]} and #{data[1]}"
+        if data[2].empty? or data[2][:conditions].empty?
+          data[2] = {:conditions => conditions}
+        else                      
+          data[2][:conditions] += " AND #{conditions}"
+        end                             
+        data[2][:select] = "#{table_name}.*"
+        model_class.find(:all, data[2]).each do |ar_object|
+          begin
+            if data[4].kind_of?(String)
+              begin
+                data[4].constantize.each(ar_object)
+              rescue NameError
+                if ar_object.respond_to?(data[4].to_sym)
+                  ar_object.send(data[4])
+                else
+                  raise NameError.new("#{data[4]} is not a class or an instance method in #{model_class}")
+                end
+              end
+            else
+              data[4].call(ar_object)
+            end
+          rescue Exception => e
+            if data[4].kind_of?(String)
+              log.error("Error in #{data[4]} #{e.inspect} #{e.backtrace.join("\n")}")              
+            else
+              log.error("Error in #{self} with given block #{e.inspect} #{e.backtrace.join("\n")}")
+            end
+          end
+        end
+      end    
+      nil
+    end
+  end
+end