brendan-skynet 0.9.3.1

data/lib/skynet/message_queue_adapters/tuple_space.rb

@@ -0,0 +1,316 @@
+require 'rinda/tuplespace'
+class Rinda::TupleSpaceProxy
+  def take(tuple, sec=nil, &block)
+    port = []
+    port.push @ts.move(nil, tuple, sec, &block)
+    port[0]
+  end
+end
+
+class Skynet
+  class Error < StandardError
+  end
+
+  class RequestExpiredError < Skynet::Error
+	end
+
+	class InvalidMessage < Skynet::Error
+  end
+
+  class MessageQueueAdapter
+
+    class TupleSpace < Skynet::MessageQueueAdapter
+
+      include SkynetDebugger
+
+      USE_FALLBACK_TASKS = true
+
+      @@ts = nil
+      @@curhostidx = 0
+
+      def self.debug_class_desc
+        "TUPLESPACE"
+      end
+
+      def self.adapter
+        :tuplespace
+      end
+
+      attr_accessor :start_options
+      
+      def initialize(options={})
+        @start_options = options
+        @ts = self.class.get_tuple_space(options)
+      end
+
+      def take_next_task(curver,timeout=nil,payload_type=nil,queue_id=0)
+        message = Skynet::Message.new(take(Skynet::Message.next_task_template(curver,payload_type, queue_id),timeout))
+        write_fallback_task(message)
+        message
+      end
+
+      def write_message(message,timeout=nil)
+        timeout ||= message.expiry
+        write(message,timeout)
+      end
+
+      def write_result(message,result=[],timeout=nil)
+        result_message = message.result_message(result).to_a
+        timeout ||= result_message.expiry
+        write(result_message,timeout)
+        take_fallback_message(message)
+        result_message
+      end
+
+      def take_result(job_id,timeout=nil)
+        Skynet::Message.new(take(Skynet::Message.result_template(job_id),timeout))
+      end
+
+      def write_error(message,error='',timeout=nil)
+        timeout ||= message.expiry
+        write(message.error_message(error),timeout)
+        take_fallback_message(message)
+      end
+
+      def list_tasks(iteration=nil,queue_id=0)
+        read_all(Skynet::Message.outstanding_tasks_template(iteration,queue_id))
+      end
+
+      def list_results
+        read_all(Skynet::Message.outstanding_results_template)
+      end
+
+      def version_active?(curver=nil, queue_id= 0)
+        return true unless curver
+        begin
+          message_row = read(Skynet::Message.next_task_template(curver, nil, queue_id),0.00001)
+          true
+        rescue Skynet::RequestExpiredError
+          return true if curver.to_i == get_worker_version.to_i
+          false
+        end
+      end
+
+      def get_worker_version
+        begin
+          message = Skynet::WorkerVersionMessage.new(read(Skynet::WorkerVersionMessage.template,0.00001))
+          if message
+            curver = message.version
+          else
+            curver=0
+          end
+        rescue Skynet::RequestExpiredError => e
+          curver = 0
+        end
+        curver
+      end
+
+      def set_worker_version(ver=nil)
+        begin
+          messages = read_all(Skynet::WorkerVersionMessage.template).collect {|ret| Skynet::WorkerVersionMessage.new(ret)}
+          curver = 0
+          messages.each do |message|
+            curver = message.version
+            debug "CURRENT WORKER VERSION #{curver}"
+            curvmessage = Skynet::WorkerVersionMessage.new(take(message.template,0.00001))
+            if curvmessage
+              curver = curvmessage.version
+            else
+              curver=0
+            end
+          end
+        rescue Skynet::RequestExpiredError => e
+          curver = 0
+        end
+
+        newver = ver ? ver : curver + 1
+        debug "WRITING CURRENT WORKER REV #{newver}"
+        write(Skynet::WorkerVersionMessage.new(:version=>newver))
+        newver
+      end
+
+      def clear_outstanding_tasks
+        begin
+          tasks = read_all(Skynet::Message.outstanding_tasks_template)
+        rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
+          error "ERROR #{e.inspect}", caller
+        end
+
+        tasks.size.times do |ii|
+          take(Skynet::Message.outstanding_tasks_template,0.00001)
+        end
+
+        results = read_all(Skynet::Message.outstanding_results_template)
+        results.size.times do |ii|
+          take(Skynet::Message.outstanding_results_template,0.00001)
+        end
+
+        task_tuples = read_all(Skynet::Message.outstanding_tasks_template)
+        result_tuples = read_all(Skynet::Message.outstanding_results_template)
+        return task_tuples + result_tuples
+      end
+
+      def stats
+        t1 = Time.now
+        tasks = list_tasks
+        results = list_results
+        t2 = Time.now - t1
+        p_tasks = tasks.partition {|task| task[9] == 0}
+        {:taken_tasks => p_tasks[1].size, :untaken_tasks => p_tasks[0].size, :results => list_results.size, :time => t2.to_f}
+      end
+
+      private
+
+      attr_accessor :ts
+
+      def write(tuple,timeout=nil)
+        ts_command(:write,tuple,timeout)
+      end
+
+      def take(template,timeout=nil)
+        ts_command(:take,template,timeout)
+      end
+
+      def read(template,timeout=nil)
+        ts_command(:read,template,timeout)
+      end
+
+      def read_all(template)
+        ts_command(:read_all,template)
+      end
+
+      ###### FALLBACK METHODS
+      def write_fallback_task(message)
+        return unless USE_FALLBACK_TASKS
+        debug "4 WRITING BACKUP TASK #{message.task_id}", message.to_h
+        ftm = message.fallback_task_message
+        debug "WRITE FALLBACK TASK", ftm.to_h
+        timeout = message.expiry * 8
+        write(ftm,timeout) unless ftm.iteration == -1
+        ftm
+      end
+
+      def take_fallback_message(message,timeout=0.01)
+        return unless USE_FALLBACK_TASKS
+        return if message.retry <= message.iteration
+        begin
+          # debug "LOOKING FOR FALLBACK TEMPLATE", message.fallback_template
+          fb_message = Skynet::Message.new(take(message.fallback_template,timeout))
+          # debug "TOOK FALLBACK MESSAGE for TASKID: #{fb_message.task_id}"
+        rescue Skynet::RequestExpiredError => e
+          error "Couldn't find expected FALLBACK MESSAGE", Skynet::Message.new(message.fallback_template).to_h
+        end
+      end
+      ## END FALLBACK METHODS
+
+      def ts_command(command,message,timeout=nil)
+        # tries = 0
+        # until(tries > 3)
+        if message.is_a?(Skynet::Message)
+          tuple = message.to_a
+        elsif message.is_a?(Array)
+          tuple = message
+        else
+          raise InvalidMessage.new("You must provide a valid Skynet::Message object when calling #{command}.  You passed #{message.inspect}.")
+        end
+
+        begin
+          if command==:read_all
+            return ts.send(command,tuple)
+          else
+            return ts.send(command,tuple,timeout)
+          end
+
+        rescue Rinda::RequestExpiredError
+          raise Skynet::RequestExpiredError.new
+        rescue DRb::DRbConnError => e
+          begin
+            error "Couldnt run command [#{command}] on tuplespace. start options: #{@start_options.inspect}"
+            @ts = self.class.get_tuple_space(@start_options)
+            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect} #{@start_options.inspect}")
+            # tries += 1
+            # next
+          rescue Skynet::ConnectionError => e
+            raise Skynet::ConnectionError.new("Can't find ring finger. #{e.inspect} #{@start_options.inspect}")
+          # rescue RuntimeError => e
+          #   raise Skynet::ConnectionError.new("Can't find ring finger. #{}")
+          rescue DRb::DRbConnError, Errno::ECONNREFUSED => e
+            raise Skynet::ConnectionError.new("There was a problem conected to the #{self.class} #{e.class} #{e.message}")
+          end
+        end
+        # end
+      end
+
+  ####################################
+  ######## CLASS METHODS #############
+  ####################################
+
+      ### XXX ACCEPT MULTIPLE TUPLE SPACES and a flag whether to use replication or failover.
+
+      def self.get_tuple_space(options = {})    
+        use_ringserver   = options[:use_ringserver]
+        ringserver_hosts = options[:ringserver_hosts]
+        drburi           = options[:drburi]
+        
+        return @@ts if valid_tuplespace?(@@ts)
+        loop do
+          begin
+            DRb.start_service
+            if use_ringserver
+              ringserver_hosts[@@curhostidx] =~ /(.+):(\d+)/
+              host = $1
+              port = $2.to_i
+              @@ts = connect_to_tuple_space(host,port,use_ringserver)
+            else
+              drburi = "druby://#{drburi}" unless drburi =~ %r{druby://}
+              @@ts = get_tuple_space_from_drburi(drburi)
+              raise DRb::DRbConnError.new unless valid_tuplespace?(@@ts)
+              info "#{self} CONNECTED TO #{drburi}"
+            end
+            return @@ts
+          rescue RuntimeError => e
+            if ringserver_hosts[@@curhostidx + 1]
+              error "#{self} Couldn't connect to #{ringserver_hosts[@@curhostidx]} trying #{ringserver_hosts[@@curhostidx+1]}"
+              @@curhostidx += 1
+              next
+            else
+              raise Skynet::ConnectionError.new("Can't find ring finger @ #{ringserver_hosts[@@curhostidx]}. #{e.class} #{e.message}")
+            end
+          rescue Exception => e
+            raise Skynet::ConnectionError.new("Error getting tuplespace @ #{ringserver_hosts[@@curhostidx]}. #{e.class} #{e.message}")
+          end
+        end
+        return @@ts
+      end
+
+      def self.connect_to_tuple_space(host,port,use_ringserver=Skynet::CONFIG[:TS_USE_RINGSERVER])
+        info "#{self} trying to connect to #{host}:#{port}"
+        if use_ringserver
+          ring_finger = Rinda::RingFinger.new(host,port)
+          ring_server = ring_finger.lookup_ring_any(0.5)
+
+          ringts = ring_server.read([:name, :TupleSpace, nil, nil],0.00005)[2]
+          ts = Rinda::TupleSpaceProxy.new(ringts)
+        else
+          ts = get_tuple_space_from_drburi("druby://#{host}:#{port}")
+        end
+        info "#{self} CONNECTED TO #{host}:#{port}"
+        ts
+      end
+
+      def self.get_tuple_space_from_drburi(drburi)
+        DRbObject.new(nil, drburi)
+      end
+
+      def self.valid_tuplespace?(ts)
+        return false unless ts
+        begin
+          ts.read_all([:valid])
+          return true
+        rescue DRb::DRbConnError, RuntimeError, Errno::ECONNREFUSED  => e
+          return false
+        end
+      end
+    end
+  end
+end

data/lib/skynet/skynet_active_record_extensions.rb

@@ -0,0 +1,293 @@
+module Skynet::ActiveRecordExtensions
+  
+  module ClassMethods
+    
+    def distributed_find(*args)
+      all = ActiveRecord::Mapreduce.find(*args)
+      all.model_class = self
+      all
+    end
+    
+  end
+  
+  def send_later(method,*arguments)
+    raise NoMethodError.new("Method: #{method} doesn't exist in #{self.class}") unless self.respond_to?(method)
+    data = { 
+      :model_class => self.class.to_s, 
+      :model_id => self.id, 
+      :method => method,       
+    }
+    # data[:save] = 1 if save
+    data[:opts] = arguments.to_yaml unless arguments.empty?
+    
+    jobopts = {
+      :single                => true,
+      :mappers               => 1,
+      :map_data              => [data],
+      :name                  => "send_later #{self.class}##{method}",
+      :map_name              => "",
+      :map_timeout           => 60,
+      :reduce_timeout        => 60,
+      :master_timeout        => 60,
+      :master_result_timeout => 1.minute,
+      :map_reduce_class      =>  Skynet::ActiveRecordAsync,
+      :master_retry          => 0,
+      :map_retry             => 0
+    }   
+    job = Skynet::AsyncJob.new(jobopts)
+    job.run
+  end
+  
+  private
+  
+  def method_missing(method_id, *arguments, &block)
+    return super unless method_id.to_s =~ /^(.*)_later$/
+    send_later($1, *arguments)
+  end
+  
+end
+
+ActiveRecord::Base.send :include, Skynet::ActiveRecordExtensions
+ActiveRecord::Base.extend Skynet::ActiveRecordExtensions::ClassMethods
+
+class Skynet::ActiveRecordAsync
+  include SkynetDebugger
+  
+  def self.map(datas)
+    datas.each do |data| 
+      begin            
+        model = data[:model_class].constantize.find(data[:model_id])
+        if data.key?(:opts)
+          model.send(data[:method], *YAML.load(data[:opts]))
+        else
+          model.send(data[:method])
+        end
+        model.save if data[:save]
+      rescue Exception => e
+        error "Error in #{self} #{e.inspect} with data #{data.inspect}"
+      end
+    end
+    return
+  end
+end
+
+module ActiveRecord  
+  
+  class Mapreduce
+    BATCH_SIZE=1000 unless defined?(BATCH_SIZE)
+    MAX_BATCHES_PER_JOB = 1000 unless defined?(MAX_BATCHES_PER_JOB)
+    attr_accessor :find_args, :batch_size
+    attr_reader :model_class
+    
+    delegate :primary_key, :table_name, :to => :model_klass
+    delegate :execute, :select_all, :to => 'model_klass.connection'
+    
+    def initialize(options = {})
+      @find_args = options[:find_args]
+      @batch_size = options[:batch_size] || BATCH_SIZE
+      @model_class = options[:model_class]
+    end
+
+    def model_klass
+      @model_klass ||= model_class.constantize
+    end
+
+    def model_class=(model_c)
+      @model_class = model_c.to_s
+    end
+
+    def self.find(*args)
+      if not args.first.is_a?(Hash)
+        args.shift
+      end       
+      if args.nil? or args.empty?
+        args = {} 
+      else
+        args = *args
+      end
+      new(:find_args => args, :batch_size => args.delete(:batch_size), :model_class => args.delete(:model_class))
+    end
+    
+    def log
+      Skynet::Logger.get
+    end
+
+    def each_range(opts={})
+      opts = opts.clone
+      opts[:id] || opts[:id] = 0
+      count = model_klass.count(:all,:conditions => opts[:conditions], :joins => opts[:joins])
+      if count <= batch_size
+        return yield({"first" => 0, "last" => nil, "cnt" => 0}, 0)
+      end
+      
+      rows = chunk_query(opts)            
+      # log.error "ROWS, #{rows.pretty_print_inspect}"
+      
+      
+      ii = 0                        
+      if rows.empty?
+        rows = [{"first" => 0, "last" => nil, "cnt" => ii}]
+      end
+      last_row = nil
+      while rows.any?
+        rows.each do |record| 
+          last_row = record
+          yield record, ii          
+        end
+        ii +=1                                                  
+        return if last_row["last"].nil?
+        rows = chunk_query(opts.merge(:id => rows.last["last"])) 
+      end
+      
+      if last_row["last"] and (last_row["last"].to_i - last_row["first"].to_i) >= batch_size 
+        catchall_row = {"first" => last_row["last"].to_i+1, "last" => nil, "cnt" => ii}
+        yield catchall_row, ii
+      end
+    end  
+
+    def chunk_query(opts={})
+
+      conditions = "#{table_name}.#{primary_key} > #{opts[:id]} AND ((@t1:=(@t1+1) % #{batch_size})=0)"
+      opts = opts.clone                                                               
+      if opts[:conditions].nil? or opts[:conditions].empty?
+        opts[:conditions] = conditions
+      else
+        opts[:conditions] += " AND " unless opts[:conditions].empty?
+        opts[:conditions] += conditions
+      end                
+      limit = opts[:limit] ? "LIMIT #{opts[:limit]}" : nil
+      # select @t2:=(@t2+1), @t3:=@t4, @t4:=id from profiles where ( ((@t1:=(@t1+1) % 1000)=0) or (((@t1+1) % 1000)=0) )  order by id LIMIT 100;
+
+      # BEST
+      # select @t1:=0, @t2:=0, @t3:=0, @t4:=0;
+        # select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
+        # select (@t2:=(@t2+1) % 2) as evenodd, ((@t3:=@t4)+1) as first, @t4:=id as last from profiles where ((@t1:=(@t1+1) % 1000)=0) order by id LIMIT 100;
+
+      execute('select @t1:=0, @t2:=-1, @t3:=0, @t4:=0')
+      sql = "select @t2:=(@t2+1) as cnt, ((@t3:=@t4)+1) as first, @t4:=#{table_name}.#{primary_key} as last from #{table_name} #{opts[:joins]} where #{opts[:conditions]} ORDER BY #{table_name}.#{primary_key} #{limit}"
+      # log.error "SQL #{sql}"
+      select_all(sql)
+
+      # mc.connection.select_values(mc.send(:construct_finder_sql, :select => "#{mc.table_name}.id", :joins => opts[:joins], :conditions => conditions, :limit => opts[:limit], :order => :id))
+    end
+    
+    def run_job_for_batch(batches,&block)
+      jobopts = {
+        :mappers               => 20000,
+        :map_data              => batches,
+        :name                  => "each #{model_class} MASTER",
+        :map_name              => "each #{model_class} MAP",
+        :map_timeout           => 60,
+        :master_timeout        => 12.hours,
+        :master_result_timeout => 60,        
+        :master_retry          => 0,
+        :map_retry             => 0
+      }   
+
+      job = nil
+      if block_given?
+        job = Skynet::Job.new(jobopts.merge(:map => block), :local_master => true)
+      else
+        job = Skynet::AsyncJob.new(jobopts.merge(:map_reduce_class => "#{self.class}"))
+      end         
+      job.run
+    end
+    
+    def map(klass_or_method=nil,&block)    
+      klass_or_method ||= model_class
+      log = Skynet::Logger.get
+
+      batches = []                    
+      each_range(find_args) do |ids,ii|
+        batch_item = [
+            ids['first'].to_i, 
+            ids['last'].to_i, 
+            find_args.clone,
+            model_class
+        ]
+        if block_given?          
+          batch_item << block
+        else
+          batch_item << "#{klass_or_method}" 
+        end
+        batches << batch_item
+        if batches.size >= MAX_BATCHES_PER_JOB
+          log.error "MAX BATCH SIZE EXCEEDED RUNNING: #{batches.size}"
+          run_job_for_batch(batches)
+          batches = []    
+        end
+      end
+      run_job_for_batch(batches)      
+    end
+
+    alias_method :each, :map
+    alias_method :mapreduce, :map
+
+    def model_class
+      @model_class || self.class.model_class
+    end
+    
+    def self.model_class(model_class)
+      (class << self; self; end).module_eval do
+         define_method(:model_class) {model_class}
+      end      
+    end
+    
+    def self.log
+      Skynet::Logger.get
+    end
+
+    def self.map(datas)
+      return unless datas and not datas.empty?
+      datas.each do |data|    
+        next if (not data.is_a?(Array))
+        next if data.empty?
+        model_class = data[3].constantize
+        table_name = model_class.table_name
+        conditions = "#{table_name}.#{model_class.primary_key} >= #{data[0]}"
+        conditions += " AND #{table_name}.#{model_class.primary_key} <= #{data[1]}" if data[1] > data[0]
+        conditions = "(#{conditions})"
+        # conditions = "ID BETWEEN #{data[0]} and #{data[1]}"
+        if not data[2]
+          data[2] = {:conditions => conditions}
+        elsif data[2].is_a?(Hash) and data[2].empty?
+          data[2] = {:conditions => conditions}
+        elsif data[2].is_a?(Hash) and (not data[2][:conditions] or data[2][:conditions].empty?)
+          data[2][:conditions] = conditions
+        else
+          data[2][:conditions] += " AND #{conditions}"
+        end                             
+        data[2][:select] = "#{table_name}.*"
+        
+        # log.error "GETTING #{data.pretty_print_inspect}"
+        models = model_class.find(:all, data[2])
+        # log.error "GOT MODELS: #{models.size}"
+        models.each do |ar_object|
+          begin
+            if data[4].kind_of?(String)
+              if ar_object.respond_to?(data[4].to_sym)
+                # log.error "CALLING #{data[4]} on #{ar_object.class}:#{ar_object.id}"
+                ar_object.send(data[4].to_sym)
+              else
+                begin
+                  data[4].constantize.each(ar_object)
+                rescue NameError
+                  raise NameError.new("#{data[4]} is not a class or an instance method in #{model_class}")
+                end
+              end
+            else
+              data[4].call(ar_object)
+            end
+          rescue Exception => e
+            if data[4].kind_of?(String)
+              log.error("Error in #{data[4]} #{e.inspect} #{e.backtrace.join("\n")}")              
+            else
+              log.error("Error in #{self} with given block #{e.inspect} #{e.backtrace.join("\n")}")
+            end
+          end
+        end
+      end    
+      nil
+    end
+  end
+end