scbi_mapreduce 0.0.40 → 0.0.45

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4e90686d178555e506b72d838c6124daa917ce3f
+  data.tar.gz: e67e2f68ea90141d054662425c9be5fba4001ca8
+SHA512:
+  metadata.gz: a4ab49d1a862ca5432744b2c1b6039dfd6de3c6b391e163cf92713f2b49ea647b2aecc6561b76ca27523fa234f515d37f4ad3aa764db7ef219e15667ce2790ab
+  data.tar.gz: 414bd290d851360a7b2d8e4cfd6439d4aa7b46e3eb06798f856f6265992dd94186f69ddbed6dc3faa4b758d821aa35752b1b86fad45b32ea29fd28830ac74fbf

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in scbi_mapreduce.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 dariogf
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,28 +1,8 @@
-require 'rubygems'
-gem 'hoe', '>= 2.1.0'
-require 'hoe'
-require 'fileutils'
-require './lib/scbi_mapreduce'
-
-Hoe.plugin :newgem
-# Hoe.plugin :website
-# Hoe.plugin :cucumberfeatures
-
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
-$hoe = Hoe.spec 'scbi_mapreduce' do
-  self.developer 'Dario Guerrero', 'dariogf@gmail.com'
-  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
-  self.rubyforge_name       = self.name # TODO this is default value
-  # self.extra_deps         = [['activesupport','>= 2.0.2']]
-  self.extra_deps         = [['eventmachine','>= 0.12.0'],['json','>= 0']]
-
-
-end
-
-require 'newgem/tasks'
-Dir['tasks/**/*.rake'].each { |t| load t }
-
-# TODO - want other tests/tasks run by default? Add them to the list
-# remove_task :default
-# task :default => [:spec, :features]
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = "test/*_test.rb"
+end

data/lib/scbi_mapreduce.rb

@@ -1,15 +1,7 @@
-$:.unshift(File.dirname(__FILE__)) unless
-$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
-
-# $: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
-
-$: << File.expand_path('scbi_mapreduce')
-# puts $:
+require "scbi_mapreduce/version"
 
 module ScbiMapreduce
-   VERSION = '0.0.40'
-   
-   
+  # Your code goes here...
 end
 
 class Time

data/lib/scbi_mapreduce/main_worker.rb

@@ -35,17 +35,31 @@ ip = ARGV[1]
 port = ARGV[2].to_i
 custom_worker_file = ARGV[3]
 
-puts "Launching worker with: worker_id:#{worker_id}, ip:#{ip}, port:#{port}, worker_file:#{custom_worker_file}"
+using_slurm=false
 
-#$: << File.expand_path(File.dirname(custom_worker_file))
+if worker_id.upcase == 'AUTO'
+  worker_id = ENV['SLURM_PROCID']
+  using_slurm=true
+end
+
+if worker_id.to_i == 0 && using_slurm
+  puts "Launching worker with: worker_id:#{worker_id}, ip:#{ip}, port:#{port}, worker_file:#{custom_worker_file}"
+  puts "Ignoring first worker in manager node worker_id:#{worker_id}"
+else
+
+  puts "Launching worker with: worker_id:#{worker_id}, ip:#{ip}, port:#{port}, worker_file:#{custom_worker_file}"
 
-require custom_worker_file
+  #$: << File.expand_path(File.dirname(custom_worker_file))
 
-klass_name = File.basename(custom_worker_file,File.extname(custom_worker_file)).camelize
+  require custom_worker_file
 
-worker_class = Object.const_get(klass_name)
+  klass_name = File.basename(custom_worker_file,File.extname(custom_worker_file)).camelize
 
-worker_class.start_worker(worker_id,ip,port)
+  worker_class = Object.const_get(klass_name)
+
+  worker_class.start_worker(worker_id,ip,port)
+
+end
 
 puts "FINISH WORKER"
 

data/lib/scbi_mapreduce/manager.rb

@@ -96,12 +96,16 @@ module ScbiMapreduce
 
     end
 
+    
+
+
     #  Start a EventMachine loop acting as a server for incoming workers connections
     def start_server
 
       # set a custom error handler, otherwise errors are silently ignored when they occurs inside a callback.
       EM.error_handler{ |e|
         $SERVER_LOG.error(e.message + ' => ' + e.backtrace.join("\n"))
+        @work_manager_class.global_error_received(e)
       }
       
       # $SERVER_LOG.info("Installing INT and TERM traps in #{@work_manager_class}")

data/lib/scbi_mapreduce/version.rb

@@ -0,0 +1,3 @@
+module ScbiMapreduce
+  VERSION = "0.0.45"
+end

data/lib/scbi_mapreduce/work_manager.rb

@@ -148,6 +148,14 @@ module ScbiMapreduce
 
     end
 
+    def self.work_manager_finished
+
+    end
+
+    def self.global_error_received(error_exception)
+
+    end
+
     def next_work
 
     end
@@ -385,6 +393,7 @@ module ScbiMapreduce
         
         t=Time.now_us
         
+        begin
         # prepare new data
         @@chunk_size.times do
           obj=next_work
@@ -395,6 +404,13 @@ module ScbiMapreduce
             objs << obj
           end
         end
+        rescue Exception => e
+          $SERVER_LOG.error("Exception creating next_work. Worker, quit!")
+          send_object(:sleep)
+          self.class.global_error_received(e)
+
+          #raise e
+        end
         
         @@total_read_time+=(Time.now_us - t)
         
@@ -648,8 +664,10 @@ module ScbiMapreduce
       EM.stop
       $SERVER_LOG.info  "Exiting server"
 
+
       self.class.end_work_manager
 
+
       @@total_seconds = (Time.now_us-@@total_seconds)
       @@total_manager_time= @@total_manager_time 
       
@@ -679,6 +697,8 @@ module ScbiMapreduce
       @@stats[:connected_workers]=@@max_workers
       @@stats[:each_transmission_time]=@@each_transmission_time
       @@stats[:each_worker_time]=@@each_worker_time
+
+
       
       
       $SERVER_LOG.info  "Total processed: #{@@count} objects in #{@@total_seconds} seconds"
@@ -703,6 +723,8 @@ module ScbiMapreduce
       $SERVER_LOG.info  "Chunk size: #{@@chunk_size}"
       $SERVER_LOG.info  "Total connected workers: #{@@max_workers}"
       
+      self.class.work_manager_finished
+
     end
 
   end

data/lib/scbi_mapreduce/worker_launcher.rb

@@ -40,6 +40,34 @@ module ScbiMapreduce
     end
 
     def launch_workers
+
+      if system("which srun > /dev/null 2>&1") && (!ENV['SLURM_PROCID'].to_s.empty?)
+          $LAUNCHER_LOG.info "SLURM DETECTED"
+          $LAUNCHER_LOG.info "Launching #{@workers} workers via srun"
+          launch_workers_srun
+      else
+          $LAUNCHER_LOG.info "Launching #{@workers} workers via SSH"
+          launch_workers_ssh
+      end
+
+    end
+
+    def launch_workers_srun
+      # TODO - si aqui falla algo, no peta, se bloquea
+      $LAUNCHER_LOG.info "Launching #{@workers} srun workers"
+        
+        pid=fork{
+          $LAUNCHER_LOG.info "Connecting #{@workers} srun workers to #{@server_ip}:#{@server_port}"
+          cmd = "srun #{File.join(File.dirname(__FILE__),'main_worker.rb')} auto #{server_ip} #{server_port} #{@worker_file}"
+          $LAUNCHER_LOG.info cmd
+          exec(cmd)
+        }
+
+        $LAUNCHER_LOG.info "All workers launched"
+    
+    end
+
+    def launch_workers_ssh
       # TODO - si aqui falla algo, no peta, se bloquea
       $LAUNCHER_LOG.info "Launching #{@workers} local workers"
       if @workers > 0
@@ -111,6 +139,12 @@ module ScbiMapreduce
 
 
     def launch_external_workers(workers)
+
+      #skip if slurm detected
+      if system("which srun > /dev/null 2>&1")
+        return
+      end
+
       puts "Launching #{workers.count} external workers: #{workers}"
       puts "INIT_ENV_FILE: #{@init_env_file}"
       

data/scbi_mapreduce.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scbi_mapreduce/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "scbi_mapreduce"
+  spec.version       = ScbiMapreduce::VERSION
+  spec.authors       = ["dariogf"]
+  spec.email         = ["dariogf@gmail.com"]
+  spec.summary       = %q{scbi_mapreduce brings parallel and distributed computing capabilities to your code.}
+  spec.description   = %q{scbi_mapreduce brings parallel and distributed computing capabilities to your code, with a very easy to use framework that allows you to exploit your clustered or cloud computational resources.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_runtime_dependency 'eventmachine','>=0.12.0'
+  spec.add_runtime_dependency 'json','>=0'
+
+end

data/skeleton/old/dummy_calculations/README.txt

@@ -0,0 +1,25 @@
+Comparison of workers with scbi_mapreduce vs ruby-threads
+=========================================================
+
+This application is only useful for testing. You can modify the files
+to perform other tasks. There are other templates available, you 
+can list them by issuing this command:
+
+scbi_mapreduce
+
+You can launch the tests application right now with the following command:
+
+  time ruby main.rb
+  
+  
+This launches 4 workers that do some simple calculations (only to keep busy 
+the processor), to demonstrate the gain speed agains threads. 4 workers are 
+used for a quad-core processor. Adjust it accordingly to your processor cores. 
+
+
+To launch the threaded version of the application, you can do:
+
+  time ruby threads_implementation.rb
+  
+You can compare the two times obtained. Threaded version will last the same with 1 thread or with 100.
+

data/skeleton/old/dummy_calculations/lib/calculations.rb

@@ -0,0 +1,37 @@
+module Calculations
+  
+  
+  def do_dummy_calculations
+    t=Time.now
+    x=0
+  	20000000.times do |i|
+		x+=1
+	end
+    puts Time.now-t
+  end
+
+  def do_dummy_calculations2
+    numer_of_calcs=250000
+
+    # t=Time.now
+
+    x1=1
+    x2=1
+
+    # do a loop with calculations
+    numer_of_calcs.times do |i|
+      x=x1+x2
+
+      x1=x2
+      x2=x
+
+      # puts some info at regular intervals
+      # if (i % 100000)==0
+      #   puts "Calculated #{i}"
+      # end
+    end
+    # puts Time.now-t
+
+  end
+
+end

data/skeleton/old/dummy_calculations/lib/thread_pool.rb

@@ -0,0 +1,107 @@
+require "thread.rb"
+
+######################################
+# This class creates a thread's pool
+######################################
+
+class ThreadPool
+  class Worker
+    @@count=0
+    def initialize
+
+      @identifier = @@count
+      @@count+=1
+      
+      Thread.abort_on_exception = true
+      @mutex = Mutex.new
+      @thread = Thread.new do
+          while true
+            sleep 0.001
+              block = get_block
+              if block
+                begin        
+                  block.call
+                rescue Exception => e
+                  puts "In thread: " + e.message
+                  raise e
+                end
+                
+                reset_block
+              end
+          end
+      end
+    end
+    
+    def get_block
+      @mutex.synchronize {@block}
+    end
+    
+    def set_block(block)
+      # puts "set block #{@identifier}"
+      @mutex.synchronize do
+        raise RuntimeError, "Thread already busy." if @block
+        @block = block
+      end
+    end
+    
+    def reset_block
+      @mutex.synchronize {@block = nil}
+    end
+    
+    def busy?
+      @mutex.synchronize {!@block.nil?}
+    end
+  end
+  
+  attr_accessor :max_size
+  attr_reader :workers
+
+  # Defines the max number of threads that will be able to exist
+  def initialize(max_size = 10)
+    @max_size = max_size
+    @workers = []
+    @mutex = Mutex.new
+  end
+  
+  def size
+    @mutex.synchronize {@workers.size}
+  end
+  
+  def busy?
+    @mutex.synchronize {@workers.any? {|w| w.busy?}}
+  end
+  
+  #Allows that main program doesn't finish until the thread have been executed
+  def join
+    sleep 0.01 while busy?
+  end
+  
+  # Begin the block's processing. After using this method, will call to "join"
+  def process(&block)
+    wait_for_worker.set_block(block)
+  end
+  
+  def wait_for_worker
+    while true
+      worker = find_available_worker
+      return worker if worker
+      sleep 0.01
+    end
+  end
+  
+  def find_available_worker
+    @mutex.synchronize {free_worker || create_worker}
+  end
+  
+  def free_worker
+    @workers.each {|w| return w unless w.busy?}; nil
+  end
+  
+  def create_worker
+    return nil if @workers.size >= @max_size
+    worker = Worker.new
+    @workers << worker
+    worker
+  end
+  private :wait_for_worker , :find_available_worker , :free_worker , :create_worker
+end