scbi_mapreduce 0.0.29

data/lib/scbi_mapreduce/worker.rb

@@ -0,0 +1,163 @@
+#!/usr/bin/env ruby
+
+require 'eventmachine'
+require 'logger'
+
+# require 'error_handler'
+
+module ScbiMapreduce
+
+  class Worker < EventMachine::Connection
+    include EM::P::ObjectProtocol
+
+
+
+    def receive_initial_config(obj)
+
+
+    end
+
+
+    def process_object(obj)
+
+    end
+
+
+    def starting_worker
+
+
+    end
+
+    def worker_connected
+
+    end
+
+    def closing_worker
+
+
+    end
+    ######################
+
+    def initialize(*args)
+      super
+
+    end
+
+    def post_init
+      $WORKER_LOG.info('WORKER CONNECTED')
+
+      worker_connected
+    rescue Exception => e
+      $WORKER_LOG.error("Exiting worker #{@@worker_id} due to exception:\n"	+ e.message+"\n"+e.backtrace.join("\n"))
+      #raise e
+    end
+
+    def receive_object(obj)
+
+      if @@count < 0
+        @@count += 1
+        # receive initial config
+        if obj != :no_initial_config then
+          receive_initial_config(obj[:initial_config])
+
+          $WORKER_LOG.info('Initial config: received')
+        else
+          $WORKER_LOG.info('Initial config: empty config')
+        end
+        # At first iteration, start worker
+        starting_worker
+      else
+
+        if obj == :quit
+          stop_worker
+        else
+          @@count += 1
+
+          # OJO  - HAY QUE PASAR EL MODIFIED OBJECT
+          #	 operation = proc {
+          #	 							# calculations
+          #								obj=process_object(obj)
+          #								#puts '.' + obj.seq_name
+          #								#return obj
+          #							}
+          #
+          #							callback = proc { |modified_obj|
+          #								send_object(modified_obj)
+          #							}
+          #
+          #							EventMachine.defer(operation, callback)
+          #send_object(obj)
+
+
+          begin
+
+            modified_data=process_object(obj.data)
+            obj.data = modified_data
+
+            send_object(obj)
+
+          rescue Exception => e
+            $WORKER_LOG.error("Error processing object\n" + e.message + ":\n" + e.backtrace.join("\n"))
+            exception= WorkerError.new('Message',e,@@worker_id,obj)
+            send_object(exception)
+
+          end
+
+
+        end
+      end
+    end
+
+    def unbind
+      $WORKER_LOG.info "EXITING WORKER"
+      EventMachine::stop_event_loop
+    end
+
+    def stop_worker
+      close_connection
+      EventMachine::stop_event_loop
+      closing_worker
+    end
+
+    def self.start_worker(worker_id,ip,port,log_file=nil)
+      #puts "NEW WORKER - INIIIIIIIIIIIIIIIIIIIIT #{self}"
+      ip = ip
+      port = port
+      @@count = -1
+
+      @@worker_id=worker_id
+
+      if log_file.nil?
+        log_file = 'logs/worker'+worker_id+'_'+`hostname`.chomp+'_log.txt'
+      end
+
+      FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
+
+      $WORKER_LOG = Logger.new(log_file)
+      $WORKER_LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
+
+      $LOG = $WORKER_LOG
+
+      total_seconds = Time.now
+
+      EM.error_handler{ |e|
+        $WORKER_LOG.error(e.message + ' => ' + e.backtrace.join("\n"))
+      }
+
+      EventMachine::run {
+
+        EventMachine::connect ip, port, self
+        $WORKER_LOG.info "Worker connected to #{ip}:#{port}"
+
+      }
+
+      total_seconds = Time.now-total_seconds
+      $WORKER_LOG.info "Client #{@@worker_id} processed: #{@@count} objs"
+      $WORKER_LOG.info "Client #{@@worker_id} proc rate: #{@@count/total_seconds.to_f} objects/seg"
+
+    end
+
+
+  end
+
+end

data/lib/scbi_mapreduce/worker_launcher.rb

@@ -0,0 +1,96 @@
+module ScbiMapreduce
+
+  INTERPRETER='ruby'
+
+  class WorkerLauncher
+
+    attr_accessor :server_ip, :server_port
+
+    def initialize(server_ip,server_port,workers, worker_file, log_file=nil, init_env_file=nil)
+      @server_ip = server_ip
+      @server_port = server_port
+      @worker_file = worker_file
+      @workers=workers
+      @init_env_file=init_env_file
+
+      
+      if log_file.nil?
+        
+        log_file = 'logs/launcher_log.txt'
+      end
+
+      FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
+
+
+      $LAUNCHER_LOG = Logger.new(log_file)
+
+      $LAUNCHER_LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
+    end
+
+    def launch_workers_and_wait
+      launch_workers
+      Process.waitall
+    end
+
+    def launch_workers
+      if @workers > 0
+        $LAUNCHER_LOG.info "Connecting #{@workers} local workers to #{@server_ip}:#{@server_port}"
+        threads = []
+        @workers.times do |i|
+          pid=fork{
+            launch_worker(i,@server_ip,@server_port)
+            $LAUNCHER_LOG.info "Worker #{i} launched [#{@server_ip}:#{@server_port}]"
+          }
+
+          #threads.each { |aThread|  aThread.join }
+        end
+        #Process.waitall
+        $LAUNCHER_LOG.info "All workers launched"
+      end
+    end
+
+    # override this
+    def launch_worker(worker_id, server_ip, server_port)
+
+      cmd = "#{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{server_ip} #{server_port} #{@worker_file}"
+      puts cmd
+      exec(cmd)
+    end
+
+    def launch_external_workers(workers)
+      puts "Launching #{workers.count} external workers: #{workers}"
+      worker_id=0
+      init=''
+      if @init_env_file
+        path = File.expand_path(@init_env_file)
+        # path = File.join($ROOT_PATH)
+        # puts "init_env file: #{path}"
+        if File.exists?(path)
+          puts "File #{path} exists, using it"
+          init=". #{path}; "
+        end
+      end
+
+      pwd=`pwd`.chomp
+
+      cd =''
+
+      if File.exists?(pwd)
+        cd = "cd #{pwd}; "
+      end
+
+      workers.each do |machine|
+
+        cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
+        $LAUNCHER_LOG.info cmd
+
+        pid=fork{
+          exec(cmd)
+        }
+
+        worker_id+=1
+      end
+    end
+
+  end
+end

data/lib/scbi_mapreduce/zlib_serializer.rb

@@ -0,0 +1,32 @@
+require 'zlib'
+require 'json'
+
+# A serializer class that provides compression
+# 
+# To use this instead of the default Marshal serializer, redefine the serializer method in your worker and worker_manager as this:
+# 
+# def serializer
+#   ZlibSerializer
+#   
+# end
+# 
+
+class ZlibSerializer
+
+  def self.dump(data)
+    input=Marshal.dump(data)
+    zipper = Zlib::Deflate.new(Zlib::BEST_COMPRESSION,15,9)
+    res= zipper.deflate(input, Zlib::FINISH)
+    zipper.close
+    
+    return res
+  end
+
+  def self.load(input)
+    unzipper = Zlib::Inflate.new(15)
+    res= unzipper.inflate(input)
+    unzipper.close
+
+    return Marshal.load(res)
+  end
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/scbi_mapreduce.rb'}"
+puts "Loading scbi_mapreduce gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/skeleton/dummy_calcs/README.txt

@@ -0,0 +1,25 @@
+Comparison of workers with scbi_mapreduce vs ruby-threads
+=========================================================
+
+This application is only useful for testing. You can modify the files
+to perform other tasks. There are other templates available, you 
+can list them by issuing this command:
+
+scbi_mapreduce
+
+You can launch the tests application right now with the following command:
+
+  time ruby main.rb
+  
+  
+This launches 4 workers that do some simple calculations (only to keep busy 
+the processor), to demonstrate the gain speed agains threads. 4 workers are 
+used for a quad-core processor. Adjust it accordingly to your processor cores. 
+
+
+To launch the threaded version of the application, you can do:
+
+  time ruby threads_implementation.rb
+  
+You can compare the two times obtained. Threaded version will last the same with 1 thread or with 100.
+

data/skeleton/dummy_calcs/lib/calculations.rb

@@ -0,0 +1,37 @@
+module Calculations
+  
+  
+  def do_dummy_calculations
+    t=Time.now
+    x=0
+  	20000000.times do |i|
+		x+=1
+	end
+    puts Time.now-t
+  end
+
+  def do_dummy_calculations2
+    numer_of_calcs=250000
+
+    # t=Time.now
+
+    x1=1
+    x2=1
+
+    # do a loop with calculations
+    numer_of_calcs.times do |i|
+      x=x1+x2
+
+      x1=x2
+      x2=x
+
+      # puts some info at regular intervals
+      # if (i % 100000)==0
+      #   puts "Calculated #{i}"
+      # end
+    end
+    # puts Time.now-t
+
+  end
+
+end

data/skeleton/dummy_calcs/lib/thread_pool.rb

@@ -0,0 +1,107 @@
+require "thread.rb"
+
+######################################
+# This class creates a thread's pool
+######################################
+
+class ThreadPool
+  class Worker
+    @@count=0
+    def initialize
+
+      @identifier = @@count
+      @@count+=1
+      
+      Thread.abort_on_exception = true
+      @mutex = Mutex.new
+      @thread = Thread.new do
+          while true
+            sleep 0.001
+              block = get_block
+              if block
+                begin        
+                  block.call
+                rescue Exception => e
+                  puts "In thread: " + e.message
+                  raise e
+                end
+                
+                reset_block
+              end
+          end
+      end
+    end
+    
+    def get_block
+      @mutex.synchronize {@block}
+    end
+    
+    def set_block(block)
+      # puts "set block #{@identifier}"
+      @mutex.synchronize do
+        raise RuntimeError, "Thread already busy." if @block
+        @block = block
+      end
+    end
+    
+    def reset_block
+      @mutex.synchronize {@block = nil}
+    end
+    
+    def busy?
+      @mutex.synchronize {!@block.nil?}
+    end
+  end
+  
+  attr_accessor :max_size
+  attr_reader :workers
+
+  # Defines the max number of threads that will be able to exist
+  def initialize(max_size = 10)
+    @max_size = max_size
+    @workers = []
+    @mutex = Mutex.new
+  end
+  
+  def size
+    @mutex.synchronize {@workers.size}
+  end
+  
+  def busy?
+    @mutex.synchronize {@workers.any? {|w| w.busy?}}
+  end
+  
+  #Allows that main program doesn't finish until the thread have been executed
+  def join
+    sleep 0.01 while busy?
+  end
+  
+  # Begin the block's processing. After using this method, will call to "join"
+  def process(&block)
+    wait_for_worker.set_block(block)
+  end
+  
+  def wait_for_worker
+    while true
+      worker = find_available_worker
+      return worker if worker
+      sleep 0.01
+    end
+  end
+  
+  def find_available_worker
+    @mutex.synchronize {free_worker || create_worker}
+  end
+  
+  def free_worker
+    @workers.each {|w| return w unless w.busy?}; nil
+  end
+  
+  def create_worker
+    return nil if @workers.size >= @max_size
+    worker = Worker.new
+    @workers << worker
+    worker
+  end
+  private :wait_for_worker , :find_available_worker , :free_worker , :create_worker
+end