scbi_mapreduce 0.0.31 → 0.0.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/History.txt CHANGED
@@ -1,3 +1,19 @@
1
+ === 0.0.35 2011-07-07
2
+
3
+ ip list
4
+
5
+ === 0.0.34 2011-07-07
6
+
7
+ Close ssh after worker creation with nohup
8
+
9
+ === 0.0.33 2011-06-23
10
+
11
+ Changed worker launching method to avoid process count overkill
12
+
13
+ === 0.0.32 2011-06-20
14
+
15
+ Removed ruby 1.9.2 requisite
16
+
1
17
  === 0.0.31 2011-06-17
2
18
 
3
19
  Total connected workers stats
data/Manifest.txt CHANGED
@@ -4,6 +4,7 @@ lib/scbi_mapreduce/main_worker.rb
4
4
  lib/scbi_mapreduce/manager.rb
5
5
  lib/scbi_mapreduce/work_manager.rb
6
6
  lib/scbi_mapreduce/worker.rb
7
+ lib/scbi_mapreduce/launcher.sh
7
8
  lib/scbi_mapreduce/zlib_serializer.rb
8
9
  lib/scbi_mapreduce/worker_launcher.rb
9
10
  lib/scbi_mapreduce.rb
data/Rakefile CHANGED
@@ -15,7 +15,7 @@ $hoe = Hoe.spec 'scbi_mapreduce' do
15
15
  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
16
16
  self.rubyforge_name = self.name # TODO this is default value
17
17
  # self.extra_deps = [['activesupport','>= 2.0.2']]
18
- self.extra_deps = [['eventmachine','>= 0.12.0']]
18
+ self.extra_deps = [['eventmachine','>= 0.12.0'],['json','>= 0']]
19
19
 
20
20
 
21
21
  end
@@ -0,0 +1,53 @@
1
+ #!/usr/bin/env bash
2
+
3
+ # script to launch workers with bash
4
+ # it needs to initialize ruby environment, cd to working dir, and
5
+ # get path
6
+
7
+ # function canonpath ()
8
+ # {
9
+ # echo $(cd $(dirname $1); pwd -P)/$(basename $1)
10
+ # }
11
+
12
+ absolute_path="$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"
13
+
14
+ # absolute_path=canonpath $0;
15
+ echo ABS: $absolute_path
16
+ echo HOST:
17
+ hostname
18
+ echo YO:
19
+ whoami
20
+
21
+ main_worker_path=`dirname $absolute_path`
22
+
23
+
24
+ echo $main_worker_path
25
+ if [ ! "$#" -eq "6" ] && [ ! "$#" -eq "5" ]; then
26
+ echo "Only $# parameters provided"
27
+ echo "Usage: $0 worker_id server_ip server_port worker_file init_dir init_file"
28
+ exit;
29
+ fi
30
+
31
+ worker_id=$1;
32
+ server_ip=$2;
33
+ server_port=$3;
34
+ worker_file=$4;
35
+ init_dir=$5;
36
+ init_file=$6;
37
+ echo "Launching worker: $worker_id"
38
+
39
+ if [ $init_file ] && [ -e $init_file ]; then
40
+ echo "Initializing env"
41
+ source $init_file;
42
+ fi
43
+
44
+ if [ -e $init_dir ]; then
45
+ echo "Changing to $init_dir"
46
+ cd $init_dir;
47
+ fi
48
+
49
+
50
+
51
+ echo Launching: ${main_worker_path}/main_worker.rb $worker_id $server_ip $server_port $worker_file
52
+
53
+ ${main_worker_path}/main_worker.rb $worker_id $server_ip $server_port $worker_file
@@ -33,6 +33,8 @@ ip = ARGV[1]
33
33
  port = ARGV[2].to_i
34
34
  custom_worker_file = ARGV[3]
35
35
 
36
+ puts "Launching worker with: worker_id:#{worker_id}, ip:#{ip}, port:#{port}, worker_file:#{custom_worker_file}"
37
+
36
38
  #$: << File.expand_path(File.dirname(custom_worker_file))
37
39
 
38
40
  require custom_worker_file
@@ -12,7 +12,7 @@ require 'fileutils'
12
12
 
13
13
  module ScbiMapreduce
14
14
 
15
-
15
+
16
16
 
17
17
 
18
18
  class Manager
@@ -24,16 +24,31 @@ module ScbiMapreduce
24
24
  @port=port
25
25
 
26
26
 
27
+ if log_file.nil?
28
+ log_file = File.join('logs','server_log.txt')
29
+ end
30
+
31
+ FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
32
+ $SERVER_LOG = Logger.new(log_file)
33
+
34
+
27
35
  ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
36
+ # La forma de abajo no encuentra la myrinet
37
+ # ip_list = Socket::getaddrinfo(Socket.gethostname, "echo", Socket::AF_INET).map{|x| x[3]}.uniq
38
+ ip_list << '127.0.0.1'
39
+
40
+ $SERVER_LOG.info("Available IPs: #{ip_list}")
28
41
 
29
42
  ip=ip_list.select{|one_ip| one_ip.index(server_ip)==0}.first
30
43
 
44
+
31
45
  if !ip
46
+ $SERVER_LOG.info("Ip #{server_ip} not found in available IPs: #{ip_list}")
32
47
  ip='0.0.0.0'
33
48
  end
34
-
49
+
35
50
  @ip = ip
36
-
51
+
37
52
  port = 0
38
53
 
39
54
 
@@ -60,13 +75,6 @@ module ScbiMapreduce
60
75
  @work_manager_class = work_manager_class
61
76
  @worker_launcher = WorkerLauncher.new(@ip,port,@workers,custom_worker_file,log_file,init_env_file)
62
77
 
63
-
64
- if log_file.nil?
65
- log_file = File.join('logs','server_log.txt')
66
- end
67
-
68
- FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
69
- $SERVER_LOG = Logger.new(log_file)
70
78
 
71
79
  $SERVER_LOG.info("Local workers: #{@workers}")
72
80
  $SERVER_LOG.info("Remote workers: #{@worker_names}")
@@ -1,3 +1,5 @@
1
+ # TODO - Make a per-node launcher: launch_workers server_ip server_port count
2
+
1
3
  module ScbiMapreduce
2
4
 
3
5
  INTERPRETER='ruby'
@@ -15,7 +17,6 @@ module ScbiMapreduce
15
17
 
16
18
 
17
19
  if log_file.nil?
18
-
19
20
  log_file = 'logs/launcher_log.txt'
20
21
  end
21
22
 
@@ -64,26 +65,30 @@ module ScbiMapreduce
64
65
  worker_id=0
65
66
  init=''
66
67
  if @init_env_file
67
- path = File.expand_path(@init_env_file)
68
+ init_path = File.expand_path(@init_env_file)
68
69
  # path = File.join($ROOT_PATH)
69
70
  # puts "init_env file: #{path}"
70
- if File.exists?(path)
71
- puts "File #{path} exists, using it"
72
- init=". #{path}; "
71
+ if File.exists?(init_path)
72
+ puts "File #{init_path} exists, using it"
73
+ init=". #{init_path}; "
73
74
  end
74
75
  end
75
76
 
76
- pwd=`pwd`.chomp
77
+ init_dir=Dir.pwd
77
78
 
78
79
  cd =''
79
80
 
80
- if File.exists?(pwd)
81
- cd = "cd #{pwd}; "
81
+ if File.exists?(init_dir)
82
+ cd = "cd #{init_dir}; "
82
83
  end
83
84
 
85
+ log_file=File.join(init_dir,'logs','launchers')
86
+
84
87
  workers.each do |machine|
85
88
 
86
- cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
89
+ # cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
90
+ cmd = "ssh #{machine} \"nohup #{File.join(File.dirname(__FILE__),'launcher.sh')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file} #{init_dir} #{init_path} </dev/null >> #{log_file} 2>> #{log_file} & \""
91
+
87
92
  $LAUNCHER_LOG.info cmd
88
93
 
89
94
  pid=fork{
@@ -4,7 +4,7 @@ $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname
4
4
  # $: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
5
5
 
6
6
  module ScbiMapreduce
7
- VERSION = '0.0.31'
7
+ VERSION = '0.0.35'
8
8
  end
9
9
 
10
10
  require 'scbi_mapreduce/manager'
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: scbi_mapreduce
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.0.31
5
+ version: 0.0.35
6
6
  platform: ruby
7
7
  authors:
8
8
  - Dario Guerrero
@@ -10,7 +10,7 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2011-06-17 00:00:00 Z
13
+ date: 2011-07-08 00:00:00 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: eventmachine
@@ -24,16 +24,27 @@ dependencies:
24
24
  type: :runtime
25
25
  version_requirements: *id001
26
26
  - !ruby/object:Gem::Dependency
27
- name: hoe
27
+ name: json
28
28
  prerelease: false
29
29
  requirement: &id002 !ruby/object:Gem::Requirement
30
+ none: false
31
+ requirements:
32
+ - - ">="
33
+ - !ruby/object:Gem::Version
34
+ version: "0"
35
+ type: :runtime
36
+ version_requirements: *id002
37
+ - !ruby/object:Gem::Dependency
38
+ name: hoe
39
+ prerelease: false
40
+ requirement: &id003 !ruby/object:Gem::Requirement
30
41
  none: false
31
42
  requirements:
32
43
  - - ">="
33
44
  - !ruby/object:Gem::Version
34
45
  version: 2.8.0
35
46
  type: :development
36
- version_requirements: *id002
47
+ version_requirements: *id003
37
48
  description: scbi_mapreduce brings parallel and distributed computing capabilities to your code, with a very easy to use framework that allows you to exploit your clustered or cloud computational resources.
38
49
  email:
39
50
  - dariogf@gmail.com
@@ -55,6 +66,7 @@ files:
55
66
  - lib/scbi_mapreduce/manager.rb
56
67
  - lib/scbi_mapreduce/work_manager.rb
57
68
  - lib/scbi_mapreduce/worker.rb
69
+ - lib/scbi_mapreduce/launcher.sh
58
70
  - lib/scbi_mapreduce/zlib_serializer.rb
59
71
  - lib/scbi_mapreduce/worker_launcher.rb
60
72
  - lib/scbi_mapreduce.rb