scbi_mapreduce 0.0.31 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +16 -0
- data/Manifest.txt +1 -0
- data/Rakefile +1 -1
- data/lib/scbi_mapreduce/launcher.sh +53 -0
- data/lib/scbi_mapreduce/main_worker.rb +2 -0
- data/lib/scbi_mapreduce/manager.rb +18 -10
- data/lib/scbi_mapreduce/worker_launcher.rb +14 -9
- data/lib/scbi_mapreduce.rb +1 -1
- metadata +16 -4
data/History.txt
CHANGED
@@ -1,3 +1,19 @@
|
|
1
|
+
=== 0.0.35 2011-07-07
|
2
|
+
|
3
|
+
ip list
|
4
|
+
|
5
|
+
=== 0.0.34 2011-07-07
|
6
|
+
|
7
|
+
Close ssh after worker creation with nohup
|
8
|
+
|
9
|
+
=== 0.0.33 2011-06-23
|
10
|
+
|
11
|
+
Changed worker launching method to avoid process count overkill
|
12
|
+
|
13
|
+
=== 0.0.32 2011-06-20
|
14
|
+
|
15
|
+
Removed ruby 1.9.2 requisite
|
16
|
+
|
1
17
|
=== 0.0.31 2011-06-17
|
2
18
|
|
3
19
|
Total connected workers stats
|
data/Manifest.txt
CHANGED
@@ -4,6 +4,7 @@ lib/scbi_mapreduce/main_worker.rb
|
|
4
4
|
lib/scbi_mapreduce/manager.rb
|
5
5
|
lib/scbi_mapreduce/work_manager.rb
|
6
6
|
lib/scbi_mapreduce/worker.rb
|
7
|
+
lib/scbi_mapreduce/launcher.sh
|
7
8
|
lib/scbi_mapreduce/zlib_serializer.rb
|
8
9
|
lib/scbi_mapreduce/worker_launcher.rb
|
9
10
|
lib/scbi_mapreduce.rb
|
data/Rakefile
CHANGED
@@ -15,7 +15,7 @@ $hoe = Hoe.spec 'scbi_mapreduce' do
|
|
15
15
|
self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
|
16
16
|
self.rubyforge_name = self.name # TODO this is default value
|
17
17
|
# self.extra_deps = [['activesupport','>= 2.0.2']]
|
18
|
-
self.extra_deps = [['eventmachine','>= 0.12.0']]
|
18
|
+
self.extra_deps = [['eventmachine','>= 0.12.0'],['json','>= 0']]
|
19
19
|
|
20
20
|
|
21
21
|
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
# script to launch workers with bash
|
4
|
+
# it needs to initialize ruby environment, cd to working dir, and
|
5
|
+
# get path
|
6
|
+
|
7
|
+
# function canonpath ()
|
8
|
+
# {
|
9
|
+
# echo $(cd $(dirname $1); pwd -P)/$(basename $1)
|
10
|
+
# }
|
11
|
+
|
12
|
+
absolute_path="$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"
|
13
|
+
|
14
|
+
# absolute_path=canonpath $0;
|
15
|
+
echo ABS: $absolute_path
|
16
|
+
echo HOST:
|
17
|
+
hostname
|
18
|
+
echo YO:
|
19
|
+
whoami
|
20
|
+
|
21
|
+
main_worker_path=`dirname $absolute_path`
|
22
|
+
|
23
|
+
|
24
|
+
echo $main_worker_path
|
25
|
+
if [ ! "$#" -eq "6" ] && [ ! "$#" -eq "5" ]; then
|
26
|
+
echo "Only $# parameters provided"
|
27
|
+
echo "Usage: $0 worker_id server_ip server_port worker_file init_dir init_file"
|
28
|
+
exit;
|
29
|
+
fi
|
30
|
+
|
31
|
+
worker_id=$1;
|
32
|
+
server_ip=$2;
|
33
|
+
server_port=$3;
|
34
|
+
worker_file=$4;
|
35
|
+
init_dir=$5;
|
36
|
+
init_file=$6;
|
37
|
+
echo "Launching worker: $worker_id"
|
38
|
+
|
39
|
+
if [ $init_file ] && [ -e $init_file ]; then
|
40
|
+
echo "Initializing env"
|
41
|
+
source $init_file;
|
42
|
+
fi
|
43
|
+
|
44
|
+
if [ -e $init_dir ]; then
|
45
|
+
echo "Changing to $init_dir"
|
46
|
+
cd $init_dir;
|
47
|
+
fi
|
48
|
+
|
49
|
+
|
50
|
+
|
51
|
+
echo Launching: ${main_worker_path}/main_worker.rb $worker_id $server_ip $server_port $worker_file
|
52
|
+
|
53
|
+
${main_worker_path}/main_worker.rb $worker_id $server_ip $server_port $worker_file
|
@@ -33,6 +33,8 @@ ip = ARGV[1]
|
|
33
33
|
port = ARGV[2].to_i
|
34
34
|
custom_worker_file = ARGV[3]
|
35
35
|
|
36
|
+
puts "Launching worker with: worker_id:#{worker_id}, ip:#{ip}, port:#{port}, worker_file:#{custom_worker_file}"
|
37
|
+
|
36
38
|
#$: << File.expand_path(File.dirname(custom_worker_file))
|
37
39
|
|
38
40
|
require custom_worker_file
|
@@ -12,7 +12,7 @@ require 'fileutils'
|
|
12
12
|
|
13
13
|
module ScbiMapreduce
|
14
14
|
|
15
|
-
|
15
|
+
|
16
16
|
|
17
17
|
|
18
18
|
class Manager
|
@@ -24,16 +24,31 @@ module ScbiMapreduce
|
|
24
24
|
@port=port
|
25
25
|
|
26
26
|
|
27
|
+
if log_file.nil?
|
28
|
+
log_file = File.join('logs','server_log.txt')
|
29
|
+
end
|
30
|
+
|
31
|
+
FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
|
32
|
+
$SERVER_LOG = Logger.new(log_file)
|
33
|
+
|
34
|
+
|
27
35
|
ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
|
36
|
+
# La forma de abajo no encuentra la myrinet
|
37
|
+
# ip_list = Socket::getaddrinfo(Socket.gethostname, "echo", Socket::AF_INET).map{|x| x[3]}.uniq
|
38
|
+
ip_list << '127.0.0.1'
|
39
|
+
|
40
|
+
$SERVER_LOG.info("Available IPs: #{ip_list}")
|
28
41
|
|
29
42
|
ip=ip_list.select{|one_ip| one_ip.index(server_ip)==0}.first
|
30
43
|
|
44
|
+
|
31
45
|
if !ip
|
46
|
+
$SERVER_LOG.info("Ip #{server_ip} not found in available IPs: #{ip_list}")
|
32
47
|
ip='0.0.0.0'
|
33
48
|
end
|
34
|
-
|
49
|
+
|
35
50
|
@ip = ip
|
36
|
-
|
51
|
+
|
37
52
|
port = 0
|
38
53
|
|
39
54
|
|
@@ -60,13 +75,6 @@ module ScbiMapreduce
|
|
60
75
|
@work_manager_class = work_manager_class
|
61
76
|
@worker_launcher = WorkerLauncher.new(@ip,port,@workers,custom_worker_file,log_file,init_env_file)
|
62
77
|
|
63
|
-
|
64
|
-
if log_file.nil?
|
65
|
-
log_file = File.join('logs','server_log.txt')
|
66
|
-
end
|
67
|
-
|
68
|
-
FileUtils.mkdir_p(File.dirname(log_file)) if ((log_file!=STDOUT) && (!File.exists?(File.dirname(log_file))))
|
69
|
-
$SERVER_LOG = Logger.new(log_file)
|
70
78
|
|
71
79
|
$SERVER_LOG.info("Local workers: #{@workers}")
|
72
80
|
$SERVER_LOG.info("Remote workers: #{@worker_names}")
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# TODO - Make a per-node launcher: launch_workers server_ip server_port count
|
2
|
+
|
1
3
|
module ScbiMapreduce
|
2
4
|
|
3
5
|
INTERPRETER='ruby'
|
@@ -15,7 +17,6 @@ module ScbiMapreduce
|
|
15
17
|
|
16
18
|
|
17
19
|
if log_file.nil?
|
18
|
-
|
19
20
|
log_file = 'logs/launcher_log.txt'
|
20
21
|
end
|
21
22
|
|
@@ -64,26 +65,30 @@ module ScbiMapreduce
|
|
64
65
|
worker_id=0
|
65
66
|
init=''
|
66
67
|
if @init_env_file
|
67
|
-
|
68
|
+
init_path = File.expand_path(@init_env_file)
|
68
69
|
# path = File.join($ROOT_PATH)
|
69
70
|
# puts "init_env file: #{path}"
|
70
|
-
if File.exists?(
|
71
|
-
puts "File #{
|
72
|
-
init=". #{
|
71
|
+
if File.exists?(init_path)
|
72
|
+
puts "File #{init_path} exists, using it"
|
73
|
+
init=". #{init_path}; "
|
73
74
|
end
|
74
75
|
end
|
75
76
|
|
76
|
-
pwd
|
77
|
+
init_dir=Dir.pwd
|
77
78
|
|
78
79
|
cd =''
|
79
80
|
|
80
|
-
if File.exists?(
|
81
|
-
cd = "cd #{
|
81
|
+
if File.exists?(init_dir)
|
82
|
+
cd = "cd #{init_dir}; "
|
82
83
|
end
|
83
84
|
|
85
|
+
log_file=File.join(init_dir,'logs','launchers')
|
86
|
+
|
84
87
|
workers.each do |machine|
|
85
88
|
|
86
|
-
cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
|
89
|
+
# cmd = "ssh #{machine} \"#{init} #{cd} #{INTERPRETER} #{File.join(File.dirname(__FILE__),'main_worker.rb')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file}\""
|
90
|
+
cmd = "ssh #{machine} \"nohup #{File.join(File.dirname(__FILE__),'launcher.sh')} #{worker_id.to_s} #{@server_ip} #{@server_port} #{@worker_file} #{init_dir} #{init_path} </dev/null >> #{log_file} 2>> #{log_file} & \""
|
91
|
+
|
87
92
|
$LAUNCHER_LOG.info cmd
|
88
93
|
|
89
94
|
pid=fork{
|
data/lib/scbi_mapreduce.rb
CHANGED
@@ -4,7 +4,7 @@ $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname
|
|
4
4
|
# $: << File.join(File.dirname(__FILE__),File.basename(__FILE__,File.extname(__FILE__)))
|
5
5
|
|
6
6
|
module ScbiMapreduce
|
7
|
-
VERSION = '0.0.
|
7
|
+
VERSION = '0.0.35'
|
8
8
|
end
|
9
9
|
|
10
10
|
require 'scbi_mapreduce/manager'
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: scbi_mapreduce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.35
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Dario Guerrero
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2011-
|
13
|
+
date: 2011-07-08 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: eventmachine
|
@@ -24,16 +24,27 @@ dependencies:
|
|
24
24
|
type: :runtime
|
25
25
|
version_requirements: *id001
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
|
-
name:
|
27
|
+
name: json
|
28
28
|
prerelease: false
|
29
29
|
requirement: &id002 !ruby/object:Gem::Requirement
|
30
|
+
none: false
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: "0"
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id002
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: hoe
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
30
41
|
none: false
|
31
42
|
requirements:
|
32
43
|
- - ">="
|
33
44
|
- !ruby/object:Gem::Version
|
34
45
|
version: 2.8.0
|
35
46
|
type: :development
|
36
|
-
version_requirements: *
|
47
|
+
version_requirements: *id003
|
37
48
|
description: scbi_mapreduce brings parallel and distributed computing capabilities to your code, with a very easy to use framework that allows you to exploit your clustered or cloud computational resources.
|
38
49
|
email:
|
39
50
|
- dariogf@gmail.com
|
@@ -55,6 +66,7 @@ files:
|
|
55
66
|
- lib/scbi_mapreduce/manager.rb
|
56
67
|
- lib/scbi_mapreduce/work_manager.rb
|
57
68
|
- lib/scbi_mapreduce/worker.rb
|
69
|
+
- lib/scbi_mapreduce/launcher.sh
|
58
70
|
- lib/scbi_mapreduce/zlib_serializer.rb
|
59
71
|
- lib/scbi_mapreduce/worker_launcher.rb
|
60
72
|
- lib/scbi_mapreduce.rb
|