scbi_mapreduce 0.0.35 → 0.0.36

data/History.txt

@@ -1,6 +1,10 @@
+=== 0.0.36 2011-10-14
+
+Automatically search for a valid network if a preffered one is not provided
+
 === 0.0.35 2011-07-07
 
-ip list
+Change method to retrieve a list of available ips
 
 === 0.0.34 2011-07-07
 

data/lib/scbi_mapreduce.rb

@@ -3,8 +3,11 @@ $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname
 
 # $: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
 
+$: << File.expand_path('scbi_mapreduce')
+# puts $:
+
 module ScbiMapreduce
-   VERSION = '0.0.35'
+   VERSION = '0.0.36'
 end
 
 require 'scbi_mapreduce/manager'

data/lib/scbi_mapreduce/launcher.sh

@@ -12,10 +12,10 @@
 absolute_path="$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"
 
 # absolute_path=canonpath $0;
-echo ABS: $absolute_path
-echo HOST:
+echo Absolute path: $absolute_path
+echo Launching at HOST:
 hostname
-echo YO:
+echo With user:
 whoami
 
 main_worker_path=`dirname $absolute_path`

data/lib/scbi_mapreduce/main_worker.rb

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 
 # $: << '/Users/dariogf/progs/ruby/gems/scbi_mapreduce/lib'
+$: << File.expand_path(File.join(__FILE__,'..','..'))
+# puts $:
 
 require 'scbi_mapreduce'
 

data/lib/scbi_mapreduce/manager.rb

@@ -45,6 +45,7 @@ module ScbiMapreduce
       if !ip
         $SERVER_LOG.info("Ip #{server_ip} not found in available IPs: #{ip_list}")
         ip='0.0.0.0'
+        # gets
       end
 
       @ip = ip
@@ -62,7 +63,14 @@ module ScbiMapreduce
       @worker_names=[]
       if workers.is_a?(Integer)
         @workers=workers
-      else
+      else # workers is a file with names, or an array
+        
+        # read file
+        if workers.is_a?(String) && File.exists?(workers)
+          $SERVER_LOG.info("Loading workers file: #{workers}")
+          workers = File.read(workers).split("\n").map{|w| w.chomp}
+        end
+        
         # puts "find worker_names"
         host_name=`hostname`.chomp
         @workers=workers.count(host_name)
@@ -73,7 +81,8 @@ module ScbiMapreduce
       end
 
       @work_manager_class = work_manager_class
-      @worker_launcher = WorkerLauncher.new(@ip,port,@workers,custom_worker_file,log_file,init_env_file)
+            
+      @worker_launcher = WorkerLauncher.new(@ip,port,ip_list,@workers,custom_worker_file,log_file,init_env_file)
 
 
       $SERVER_LOG.info("Local workers: #{@workers}")

data/lib/scbi_mapreduce/worker_launcher.rb

@@ -2,20 +2,23 @@
 
 module ScbiMapreduce
 
+  require 'resolv'
+
   INTERPRETER='ruby'
 
   class WorkerLauncher
 
     attr_accessor :server_ip, :server_port
 
-    def initialize(server_ip,server_port,workers, worker_file, log_file=nil, init_env_file=nil)
+    def initialize(server_ip,server_port, server_ip_list,workers, worker_file, log_file=nil, init_env_file=nil)
       @server_ip = server_ip
       @server_port = server_port
       @worker_file = worker_file
       @workers=workers
       @init_env_file=init_env_file
+      @server_ip_list=server_ip_list
+
 
-      
       if log_file.nil?
         log_file = 'logs/launcher_log.txt'
       end
@@ -60,6 +63,50 @@ module ScbiMapreduce
       exec(cmd)
     end
 
+    def find_common_ip(machine_ip,ip_list)
+
+      def left_largest_common_substr(s1,s2)
+        res=''
+
+        s2.scan(/./).each_with_index do |l1,i|
+          if s1[i]==l1
+            res << l1
+          else
+            break
+          end
+        end
+        res
+
+      end
+
+      def remove_final_dot(s)
+        res=s
+        # remove final dot
+        if res[res.length-1]=='.'
+          res=res[0,res.length-1]
+        end
+
+        return res
+      end
+
+
+      res=''
+      common_ip=''
+
+      ip_list.each do |ip|
+
+        res=left_largest_common_substr(ip,machine_ip)
+        res=remove_final_dot(res)
+
+        if res.length>common_ip.length
+          common_ip=res
+        end
+      end
+
+      return common_ip
+    end
+
+
     def launch_external_workers(workers)
       puts "Launching #{workers.count} external workers: #{workers}"
       worker_id=0
@@ -86,16 +133,31 @@ module ScbiMapreduce
 
       workers.each do |machine|
 
-        # cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
-        cmd = "ssh #{machine} \"nohup #{File.join(File.dirname(__FILE__),'launcher.sh')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file} #{init_dir} #{init_path}  </dev/null >> #{log_file} 2>> #{log_file} & \""
-        
-        $LAUNCHER_LOG.info cmd
+        # if server_ip is not in valid ips
+        if !@server_ip_list.include?(@server_ip)
+          # find matching ip between server and worker
+          machine_ip = Resolv.getaddress(machine)
+          matching_ip=find_common_ip(machine_ip,@server_ip_list)
+          found_ip=@server_ip_list.select{|one_ip| one_ip.index(matching_ip)==0}.first
+        else
+          found_ip=@server_ip
+        end
+
+        if !found_ip.nil?
+          # cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
+          # cmd = "ssh #{machine} \"nohup #{File.join(File.dirname(__FILE__),'launcher.sh')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file} #{init_dir} #{init_path}  </dev/null >> #{log_file} 2>> #{log_file} & \""
+          cmd = "ssh #{machine} \"nohup #{File.join(File.dirname(__FILE__),'launcher.sh')} #{worker_id.to_s} #{found_ip} #{@server_port} #{@worker_file} #{init_dir} #{init_path}  </dev/null >> #{log_file} 2>> #{log_file} & \""
 
-        pid=fork{
-          exec(cmd)
-        }
+          $LAUNCHER_LOG.info cmd
 
-        worker_id+=1
+          pid=fork{
+            exec(cmd)
+          }
+
+          worker_id+=1
+        else
+          $LAUNCHER_LOG.error("Couldn't find a matching ip between worker (#{machine_ip}) and server #{ip_list.to_json}")
+        end
       end
     end
 

data/test/drb_test/main.rb

@@ -4,7 +4,7 @@ $: << File.dirname(__FILE__)
 
 require "logger"
 
-# $: << '/Users/dariogf/progs/ruby/gems/scbi_mapreduce/lib'
+$: << File.expand_path('../../lib')
 
 require 'scbi_mapreduce'
 require 'my_worker_manager'
@@ -17,6 +17,8 @@ ip='0.0.0.0'
 port = 50000
 workers = 8
 
+workers = File.expand_path('workers')
+
 custom_worker_file = File.join(File.dirname(__FILE__),'my_worker.rb')
 
 $LOG.info 'Starting server'

metadata

@@ -2,7 +2,7 @@
 name: scbi_mapreduce
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.35
+  version: 0.0.36
 platform: ruby
 authors: 
 - Dario Guerrero
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-08 00:00:00 Z
+date: 2011-10-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine