scbi_mapreduce 0.0.31 → 0.0.35

data/History.txt

@@ -1,3 +1,19 @@
+=== 0.0.35 2011-07-07
+
+ip list
+
+=== 0.0.34 2011-07-07
+
+Close ssh after worker creation with nohup
+
+=== 0.0.33 2011-06-23
+
+Changed worker launching method to avoid process count overkill
+
+=== 0.0.32 2011-06-20
+
+Removed ruby 1.9.2 requisite
+
 === 0.0.31 2011-06-17
 
 Total connected workers stats

data/Manifest.txt

@@ -4,6 +4,7 @@ lib/scbi_mapreduce/main_worker.rb
 lib/scbi_mapreduce/manager.rb
 lib/scbi_mapreduce/work_manager.rb
 lib/scbi_mapreduce/worker.rb
+lib/scbi_mapreduce/launcher.sh
 lib/scbi_mapreduce/zlib_serializer.rb
 lib/scbi_mapreduce/worker_launcher.rb
 lib/scbi_mapreduce.rb

data/Rakefile

@@ -15,7 +15,7 @@ $hoe = Hoe.spec 'scbi_mapreduce' do
   self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
   self.rubyforge_name       = self.name # TODO this is default value
   # self.extra_deps         = [['activesupport','>= 2.0.2']]
-  self.extra_deps         = [['eventmachine','>= 0.12.0']]
+  self.extra_deps         = [['eventmachine','>= 0.12.0'],['json','>= 0']]
 
 
 end

data/lib/scbi_mapreduce/launcher.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# script to launch workers with bash
+# it needs to initialize ruby environment, cd to working dir, and 
+# get path
+
+# function canonpath ()
+# { 
+#     echo $(cd $(dirname $1); pwd -P)/$(basename $1)
+# }
+
+absolute_path="$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"
+
+# absolute_path=canonpath $0;
+echo ABS: $absolute_path
+echo HOST:
+hostname
+echo YO:
+whoami
+
+main_worker_path=`dirname $absolute_path`
+
+
+echo $main_worker_path
+if [ ! "$#" -eq "6" ] && [ ! "$#" -eq "5" ]; then
+    echo "Only $# parameters provided"
+    echo "Usage: $0 worker_id server_ip server_port worker_file init_dir init_file"
+    exit;
+fi
+
+worker_id=$1;
+server_ip=$2;
+server_port=$3;
+worker_file=$4;
+init_dir=$5;
+init_file=$6;
+echo "Launching worker: $worker_id"
+
+if [ $init_file ] && [ -e $init_file ]; then
+    echo "Initializing env"
+    source $init_file;
+fi
+
+if [ -e $init_dir ]; then
+    echo "Changing to $init_dir"
+    cd $init_dir;
+fi
+
+
+
+echo Launching: ${main_worker_path}/main_worker.rb $worker_id $server_ip $server_port $worker_file
+
+${main_worker_path}/main_worker.rb $worker_id $server_ip $server_port $worker_file

data/lib/scbi_mapreduce/main_worker.rb

@@ -33,6 +33,8 @@ ip = ARGV[1]
 port = ARGV[2].to_i
 custom_worker_file = ARGV[3]
 
+puts "Launching worker with: worker_id:#{worker_id}, ip:#{ip}, port:#{port}, worker_file:#{custom_worker_file}"
+
 #$: << File.expand_path(File.dirname(custom_worker_file))
 
 require custom_worker_file

data/lib/scbi_mapreduce/manager.rb

@@ -12,7 +12,7 @@ require 'fileutils'
 
 module ScbiMapreduce
 
-  
+
 
 
   class Manager
@@ -24,16 +24,31 @@ module ScbiMapreduce
       @port=port
 
 
+      if log_file.nil?
+        log_file = File.join('logs','server_log.txt')
+      end
+
+      FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
+      $SERVER_LOG = Logger.new(log_file)
+
+
       ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
+      # La forma de abajo no encuentra la myrinet
+      # ip_list = Socket::getaddrinfo(Socket.gethostname, "echo", Socket::AF_INET).map{|x| x[3]}.uniq
+      ip_list << '127.0.0.1'
+
+      $SERVER_LOG.info("Available IPs: #{ip_list}")
 
       ip=ip_list.select{|one_ip| one_ip.index(server_ip)==0}.first
 
+
       if !ip
+        $SERVER_LOG.info("Ip #{server_ip} not found in available IPs: #{ip_list}")
         ip='0.0.0.0'
       end
-      
+
       @ip = ip
-      
+
       port = 0
 
 
@@ -60,13 +75,6 @@ module ScbiMapreduce
       @work_manager_class = work_manager_class
       @worker_launcher = WorkerLauncher.new(@ip,port,@workers,custom_worker_file,log_file,init_env_file)
 
-      
-      if log_file.nil?
-        log_file = File.join('logs','server_log.txt')
-      end
-
-      FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
-      $SERVER_LOG = Logger.new(log_file)
 
       $SERVER_LOG.info("Local workers: #{@workers}")
       $SERVER_LOG.info("Remote workers: #{@worker_names}")

data/lib/scbi_mapreduce/worker_launcher.rb

@@ -1,3 +1,5 @@
+# TODO - Make a per-node launcher: launch_workers server_ip server_port count
+
 module ScbiMapreduce
 
   INTERPRETER='ruby'
@@ -15,7 +17,6 @@ module ScbiMapreduce
 
       
       if log_file.nil?
-        
         log_file = 'logs/launcher_log.txt'
       end
 
@@ -64,26 +65,30 @@ module ScbiMapreduce
       worker_id=0
       init=''
       if @init_env_file
-        path = File.expand_path(@init_env_file)
+        init_path = File.expand_path(@init_env_file)
         # path = File.join($ROOT_PATH)
         # puts "init_env file: #{path}"
-        if File.exists?(path)
-          puts "File #{path} exists, using it"
-          init=". #{path}; "
+        if File.exists?(init_path)
+          puts "File #{init_path} exists, using it"
+          init=". #{init_path}; "
         end
       end
 
-      pwd=`pwd`.chomp
+      init_dir=Dir.pwd
 
       cd =''
 
-      if File.exists?(pwd)
-        cd = "cd #{pwd}; "
+      if File.exists?(init_dir)
+        cd = "cd #{init_dir}; "
       end
 
+      log_file=File.join(init_dir,'logs','launchers')
+
       workers.each do |machine|
 
-        cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
+        # cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
+        cmd = "ssh #{machine} \"nohup #{File.join(File.dirname(__FILE__),'launcher.sh')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file} #{init_dir} #{init_path}  </dev/null >> #{log_file} 2>> #{log_file} & \""
+        
         $LAUNCHER_LOG.info cmd
 
         pid=fork{

data/lib/scbi_mapreduce.rb

@@ -4,7 +4,7 @@ $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname
 # $: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
 
 module ScbiMapreduce
-   VERSION = '0.0.31'
+   VERSION = '0.0.35'
 end
 
 require 'scbi_mapreduce/manager'

metadata

@@ -2,7 +2,7 @@
 name: scbi_mapreduce
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.31
+  version: 0.0.35
 platform: ruby
 authors: 
 - Dario Guerrero
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-17 00:00:00 Z
+date: 2011-07-08 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine
@@ -24,16 +24,27 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: json
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
         version: 2.8.0
   type: :development
-  version_requirements: *id002
+  version_requirements: *id003
 description: scbi_mapreduce brings parallel and distributed computing capabilities to your code, with a very easy to use framework that allows you to exploit your clustered or cloud computational resources.
 email: 
 - dariogf@gmail.com
@@ -55,6 +66,7 @@ files:
 - lib/scbi_mapreduce/manager.rb
 - lib/scbi_mapreduce/work_manager.rb
 - lib/scbi_mapreduce/worker.rb
+- lib/scbi_mapreduce/launcher.sh
 - lib/scbi_mapreduce/zlib_serializer.rb
 - lib/scbi_mapreduce/worker_launcher.rb
 - lib/scbi_mapreduce.rb