scbi_mapreduce 0.0.29

data/lib/scbi_mapreduce.rb

@@ -0,0 +1,15 @@
+$:.unshift(File.dirname(__FILE__)) unless
+$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+# $: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
+
+module ScbiMapreduce
+   VERSION = '0.0.29'
+end
+
+require 'scbi_mapreduce/manager'
+require 'scbi_mapreduce/worker_launcher'
+require 'scbi_mapreduce/worker'
+require 'scbi_mapreduce/work_manager'
+require 'scbi_mapreduce/error_handler'
+require 'scbi_mapreduce/zlib_serializer'

data/lib/scbi_mapreduce/error_handler.rb

@@ -0,0 +1,15 @@
+module ScbiMapreduce
+
+  class WorkerError < Exception
+
+    attr_reader :worker_id,:original_exception, :object
+
+    def initialize(message, original_exception, worker_id, object)
+      @message = message
+      @worker_id = worker_id
+      @original_exception = original_exception
+      @object = object
+    end
+
+  end
+end

data/lib/scbi_mapreduce/main_worker.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+# $: << '/Users/dariogf/progs/ruby/gems/scbi_mapreduce/lib'
+
+require 'scbi_mapreduce'
+
+class String
+  def camelize
+    self.split(/[^a-z0-9]/i).map{|w| w.capitalize}.join
+  end
+
+  def decamelize
+    self.to_s.
+      gsub(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2').
+      gsub(/([a-z]+)([A-Z\d])/, '\1_\2').
+      gsub(/([A-Z]{2,})(\d+)/i, '\1_\2').
+      gsub(/(\d+)([a-z])/i, '\1_\2').
+      gsub(/(.+?)\&(.+?)/, '\1_&_\2').
+      gsub(/\s/, '_').downcase
+  end
+end
+
+#================= MAIN
+
+if ARGV.size != 4
+  puts "Usage #{$0} worker_id server_ip server_port custom_worker_class"
+  puts "Eg.: #{$0} 1 localhost 50000 MyWorker"
+  exit
+end
+
+worker_id = ARGV[0]
+ip = ARGV[1]
+port = ARGV[2].to_i
+custom_worker_file = ARGV[3]
+
+#$: << File.expand_path(File.dirname(custom_worker_file))
+
+require custom_worker_file
+
+klass_name = File.basename(custom_worker_file,File.extname(custom_worker_file)).camelize
+
+worker_class = Object.const_get(klass_name)
+
+worker_class.start_worker(worker_id,ip,port)
+
+puts "FINISH WORKER"
+
+
+
+# ============

data/lib/scbi_mapreduce/manager.rb

@@ -0,0 +1,110 @@
+require 'eventmachine'
+require 'socket'
+# require 'worker_launcher'
+require 'logger'
+require 'fileutils'
+
+#
+#= Manager class
+#
+# The manager side of scbi_mapreduce
+#
+
+module ScbiMapreduce
+
+  
+
+
+  class Manager
+
+    attr_accessor :checkpointing, :keep_order, :retry_failed_jobs, :exit_on_many_errors, :chunk_size
+
+    # initialize Manager
+    def initialize(server_ip, port, workers, work_manager_class,custom_worker_file,log_file=nil, init_env_file=nil)
+      @port=port
+
+
+      ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
+
+      ip=ip_list.select{|one_ip| one_ip.index(server_ip)==0}.first
+
+      if !ip
+        ip='0.0.0.0'
+      end
+      
+      @ip = ip
+      
+      port = 0
+
+
+      @checkpointing=false
+      @keep_order=false
+      @retry_failed_jobs=false
+
+      @chunk_size=1
+
+
+      @worker_names=[]
+      if workers.is_a?(Integer)
+        @workers=workers
+      else
+        # puts "find worker_names"
+        host_name=`hostname`.chomp
+        @workers=workers.count(host_name)
+
+        @worker_names=workers
+        @worker_names.delete(host_name)
+        # puts @workers
+      end
+
+      @work_manager_class = work_manager_class
+      @worker_launcher = WorkerLauncher.new(@ip,port,@workers,custom_worker_file,log_file,init_env_file)
+
+      
+      if log_file.nil?
+        log_file = File.join('logs','server_log.txt')
+      end
+
+      FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
+      $SERVER_LOG = Logger.new(log_file)
+
+
+      $SERVER_LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
+
+    end
+
+
+    #  Start a EventMachine loop acting as a server for incoming workers connections
+    def start_server
+
+      # set a custom error handler, otherwise errors are silently ignored when they occurs inside a callback.
+      EM.error_handler{ |e|
+        $SERVER_LOG.error(e.message + ' => ' + e.backtrace.join("\n"))
+      }
+
+      # start EM loop
+      EventMachine::run {
+
+        @work_manager_class.init_work_manager_internals(@checkpointing, @keep_order, @retry_failed_jobs,@exit_on_many_errors,@chunk_size)
+
+        evm=EventMachine::start_server @ip, @port, @work_manager_class
+        dir=Socket.unpack_sockaddr_in( EM.get_sockname( evm ))
+
+        @port = dir[0].to_i
+        @ip=dir[1].to_s
+
+        $SERVER_LOG.info 'Server running at : ['+@ip.to_s+':'+@port.to_s+']'
+        @worker_launcher.server_port=@port
+        @worker_launcher.launch_workers
+        @worker_launcher.launch_external_workers(@worker_names)
+
+      }
+    rescue Exception => e
+      $SERVER_LOG.error("Exiting server due to exception:\n" + e.message+"\n"+e.backtrace.join("\n"))
+      @work_manager_class.end_work_manager
+    end
+
+
+  end
+
+end

data/lib/scbi_mapreduce/work_manager.rb

@@ -0,0 +1,405 @@
+# = WorkManager
+#
+# One instance of this class is created automatically by EM to attend each worker.
+#
+#This class handles server <-> worker communications. It waits for workers connections, sends them the initial configuration parameters,
+#and later sends new jobs each time a worker request a new one until no more works are available.
+#
+# Reliability can be incremented by using a hash @@running_jobs tracking the object_id of each running work. This approach should be slower than current one.
+
+# require 'error_handler'
+
+# TODO - Data preload (queue?) instead of under demand loading
+# DONE - Add serializer with marshal + zlib deflate/inflate
+
+module ScbiMapreduce
+
+  
+  PENDING_TO_SAVE=100
+
+
+  class WorkManagerData
+
+    @@job_id=1
+
+    attr_reader :job_identifier
+    attr_accessor :status, :data
+
+    def initialize(job)
+
+      @job_identifier=@@job_id
+      @@job_id+=1
+      @data=job
+      @status=:running
+    end
+
+    def inspect
+      return "WorkManagerData: #{@job_identifier} => #{@status}"
+    end
+
+    def self.job_id=(c)
+      # puts "Setting job_id to #{c}"
+      @@job_id=c
+    end
+
+    def self.job_id
+      # puts "Setting job_id to #{c}"
+      @@job_id
+    end
+
+  end
+
+  #require 'json'
+  class WorkManager < EventMachine::Connection
+
+    include EM::P::ObjectProtocol
+    
+    def self.init_work_manager
+
+    end
+
+    def self.end_work_manager
+
+    end
+
+    def next_work
+
+    end
+
+    def work_received(obj)
+
+    end
+
+    def worker_initial_config
+
+    end
+
+    def error_received(worker_error, obj)
+
+    end
+
+    def too_many_errors_received
+
+    end
+
+    def read_until_checkpoint(checkpoint)
+
+    end
+
+    # if this function returns -1, then automatic checkpointing is done.
+    # Return 0 to no checkpointing.
+    # Return the restored checkpoint number to start in this point.
+    def load_user_checkpoint(checkpoint)
+      return -1
+    end
+
+    def save_user_checkpoint
+    end
+
+    def trash_checkpointed_work
+
+    end
+
+    ############
+
+    def self.init_work_manager_internals(checkpointing, keep_order, retry_failed_jobs,exit_on_many_errors,chunk_size)
+      @@count = 0
+      @@chunk_count = 0
+      @@workers = 0
+      @@error_count = 0
+      @@running_jobs=[]
+      # @@compress=true
+
+      @@checkpointing=checkpointing
+      @@keep_order=keep_order
+      @@retry_failed_jobs=retry_failed_jobs
+      @@exit_on_many_errors=exit_on_many_errors
+
+      # TODO - Implement a dynamic chunk_size
+
+      @@chunk_size=chunk_size
+      $SERVER_LOG.info "Processing in chunks of #{@@chunk_size} objects"
+
+      @@checkpoint=0
+      if @@checkpointing
+        @@checkpoint=self.get_checkpoint
+        $SERVER_LOG.info "Detected checkpoint at #{@@checkpoint}"
+      end
+
+    end
+
+    def self.checkpoint
+      return @@checkpoint
+    end
+
+    def save_checkpoint
+      checkpoint_file = File.open('scbi_mapreduce_checkpoint','w')
+
+      if !@@running_jobs.empty?
+        checkpoint_file.puts @@running_jobs.first.job_identifier
+      else
+        checkpoint_file.puts WorkManagerData.job_id-1
+      end
+
+      checkpoint_file.close
+
+      save_user_checkpoint
+
+    end
+
+    def self.get_checkpoint
+      res = 0
+      begin
+        if File.exists?('scbi_mapreduce_checkpoint')
+          res=File.read('scbi_mapreduce_checkpoint').chomp
+          # puts "read checkpoint #{res}"
+
+          res = res.to_i
+        end
+      rescue
+        res = 0
+      end
+
+      return res
+    end
+
+    def send_initial_config
+      config = worker_initial_config
+
+      if config.nil?
+        obj = :no_initial_config
+      else
+        obj = {:initial_config => config}
+      end
+
+      send_object(obj)
+    end
+
+    # send next work to worker
+    def send_next_work
+
+      objs=[]
+
+      @@chunk_size.times do
+        obj=next_work
+        if obj.nil?
+          break
+        else
+          # add to obj array
+          objs << obj
+        end
+      end
+
+
+      if objs.count>0
+        @@count += objs.count
+        @@chunk_count += 1
+
+        work_data=WorkManagerData.new(objs)
+
+        send_object(work_data)
+
+        # to keep order or retry failed job, we need job status
+        if @@keep_order || @@retry_failed_jobs
+          @@running_jobs.push work_data
+        end
+      else
+
+        send_object(:quit)
+      end
+
+
+    end
+
+    def goto_checkpoint
+      if @@checkpoint>0
+        $SERVER_LOG.info "Skipping until checkpoint #{@@checkpoint}"
+
+        checkpoint=load_user_checkpoint(@@checkpoint)
+
+        # do an automatic checkpoint restore
+        if checkpoint==-1
+          @@checkpoint.times do |i|
+            # puts "Skipping #{i+1}"
+
+            # get next work
+            trash_checkpointed_work
+            #            if obj
+            #             if obj.methods.include?(:count)
+            #               @@count += obj.count
+            #             else
+            #   @@count += 1
+            # end
+            #            end
+          end
+
+          $SERVER_LOG.info "Automatic checkpoint finished"
+
+          WorkManagerData.job_id=@@checkpoint
+
+          #user has done the checkpoint restoration
+        elsif checkpoint>0
+
+          WorkManagerData.job_id=checkpoint
+        elsif checkpoint==0
+          $SERVER_LOG.info "Automatic checkpoint not done"
+        end
+
+
+        @@checkpoint=0
+
+      end
+
+    end
+
+    def post_init
+      @@workers += 1
+
+      # when first worker is connected, do special config
+      if @@workers == 1
+        @@total_seconds = Time.now
+        $SERVER_LOG.info "First worker connected"
+
+        if @@checkpointing
+          $SERVER_LOG.info "Checking for checkpoint"
+          goto_checkpoint
+        end
+      end
+
+      $SERVER_LOG.info "#{@@workers} workers connected"
+      send_initial_config
+      send_next_work
+    end
+
+
+    def receive_object(obj)
+
+      # check if response is an error
+      if obj.is_a?(Exception)
+        $SERVER_LOG.error("Error in worker #{obj.worker_id} while processing object #{obj.object.inspect}\n" + obj.original_exception.message + ":\n" + obj.original_exception.backtrace.join("\n"))
+
+        @@error_count += 1
+
+        error_received(obj,obj.object.data)
+
+        # if there are too many errors
+        if (@@count>100) && (@@error_count >= @@count*0.8)
+          @@exit = @@exit_on_many_errors
+
+          # notice programmer
+          res=too_many_errors_received
+
+          # force exit if too_many_errors_received returns true
+          if res==true
+            @@exit=res
+          end
+        end
+
+      else
+        # if not using checkpointing
+
+        if @@checkpointing || @@keep_order || @@retry_failed_jobs
+          checkpointable_job_received(obj)
+        else
+          work_received(obj.data)
+        end
+      end
+
+
+      send_next_work
+
+    end
+
+
+    def checkpointable_job_received(obj)
+      received_job=@@running_jobs.find{|o| o.job_identifier==obj.job_identifier}
+
+      # save job
+      if received_job
+
+        # change job's status to received
+        received_job.data=obj.data
+        received_job.status=:received
+
+        # if there are sufficient jobs, count pending ones
+        if (@@running_jobs.count>=PENDING_TO_SAVE)
+          # count received objects pending to be written
+          pending=0
+
+          @@running_jobs.each do |job|
+            if job.status==:received
+              pending += 1
+            else
+              break
+            end
+          end
+
+
+          if (pending>PENDING_TO_SAVE) || (pending==@@running_jobs.count)
+            # purge contiguos saved data
+            to_remove = 0
+
+            @@running_jobs.each_with_index do |job,i|
+              if job.status==:received
+                # puts "Sent to save: #{job.inspect}"
+                work_received(job.data)
+                job.status=:saved
+                to_remove += 1
+              else
+                break
+              end
+            end
+
+            # if some objects were saved
+            if to_remove > 0
+              to_remove.times do |i|
+                o=@@running_jobs.shift
+                # puts "Job removed #{o.inspect}"
+              end
+
+              save_checkpoint
+            end
+          end
+        end
+      else
+        $SERVER_LOG.info "Job already processed #{obj.inspect}"
+      end
+    end
+
+    def initialize(*args)
+      super
+      #puts "WORK MANAGER INITIALIZE NEWWWWWWWWWW, ONE per worker"
+    end
+
+    # A worker has disconected
+    def unbind
+
+      @@workers -= 1
+      #puts @@running_jobs.to_json
+
+      $SERVER_LOG.info  "Worker disconnected. #{@@workers} kept running"
+
+      # no more workers left, shutdown EM and stop server
+      if @@workers == 0
+        $SERVER_LOG.info  "All workers finished"
+        EM.stop
+        $SERVER_LOG.info  "Exiting server"
+
+
+
+        self.class.end_work_manager
+
+        @@total_seconds = Time.now-@@total_seconds
+        $SERVER_LOG.info  "Total processed: #{@@count} objects in #{@@total_seconds} seconds"
+        $SERVER_LOG.info  "Processing rate: #{"%.2f" % (@@count/@@total_seconds.to_f)} objects per second"
+        $SERVER_LOG.info  "Connection rate: #{"%.2f" % (@@chunk_count/@@total_seconds.to_f)} connections per second"
+
+        $SERVER_LOG.info  "Number of errors: #{@@error_count}"
+        $SERVER_LOG.info  "Chunk size: #{@@chunk_size}"
+
+
+      end
+    end
+
+  end
+end