drbqs 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/bin/drbqs-manage +38 -10
- data/bin/drbqs-server +15 -9
- data/drbqs.gemspec +13 -4
- data/example/command/server_def.rb +14 -0
- data/example/sum/server_def.rb +5 -3
- data/example/sum/sum.rb +1 -0
- data/example/sum2/server_def.rb +27 -0
- data/example/sum2/sum.rb +9 -0
- data/lib/drbqs/config.rb +52 -0
- data/lib/drbqs/manage.rb +19 -4
- data/lib/drbqs/queue.rb +4 -1
- data/lib/drbqs/server.rb +26 -9
- data/lib/drbqs/server_define.rb +28 -18
- data/lib/drbqs/ssh_shell.rb +92 -0
- data/lib/drbqs/task.rb +26 -1
- data/lib/drbqs/task_generator.rb +19 -2
- data/lib/drbqs.rb +4 -0
- data/spec/config_spec.rb +14 -0
- data/spec/manage_spec.rb +9 -2
- data/spec/ssh_shell_spec.rb +40 -0
- metadata +67 -54
- /data/example/server/{server_def.rb → server.rb} +0 -0
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.8
|
data/bin/drbqs-manage
CHANGED
@@ -6,18 +6,38 @@ require 'optparse'
|
|
6
6
|
Version = '0.0.1'
|
7
7
|
|
8
8
|
help_message =<<HELP
|
9
|
-
Usage: #{File.basename(__FILE__)} <command>
|
9
|
+
Usage: #{File.basename(__FILE__)} <command> [arguments ...]
|
10
10
|
Manage DRbQS server by sending messages.
|
11
|
-
<command> is 'exit-signal'.
|
11
|
+
<command> is 'exit-signal', 'initialize', or 'ssh'.
|
12
12
|
|
13
13
|
HELP
|
14
14
|
|
15
|
+
command = ARGV[0]
|
16
|
+
argv, command_args = DRbQS::Manage.split_arguments(ARGV)
|
17
|
+
|
18
|
+
options = {}
|
19
|
+
|
15
20
|
begin
|
16
21
|
OptionParser.new(help_message) do |opt|
|
17
22
|
opt.on('--debug', 'Set $DEBUG true.') do |v|
|
18
23
|
$DEBUG = true
|
19
24
|
end
|
20
|
-
opt.
|
25
|
+
opt.on('--dir DIR', String, 'Set the base directory over ssh.') do |v|
|
26
|
+
options[:dir] = v
|
27
|
+
end
|
28
|
+
opt.on('--shell STR', String, 'Set the shell over ssh') do |v|
|
29
|
+
options[:shell] = v
|
30
|
+
end
|
31
|
+
opt.on('--rvm STR', String, 'Ruby version to use on RVM over ssh.') do |v|
|
32
|
+
options[:rvm] = v
|
33
|
+
end
|
34
|
+
opt.on('--rvm-init PATH', String, 'Path of script to initialize RVM over ssh.') do |v|
|
35
|
+
options[:rvm_init] = v
|
36
|
+
end
|
37
|
+
opt.on('--output PATH', String, 'File path that stdout and stderr are output to over ssh.') do |v|
|
38
|
+
options[:output] = v
|
39
|
+
end
|
40
|
+
opt.parse!(argv)
|
21
41
|
end
|
22
42
|
rescue OptionParser::InvalidOption
|
23
43
|
$stderr.print <<MES
|
@@ -33,18 +53,26 @@ MES
|
|
33
53
|
exit(2)
|
34
54
|
end
|
35
55
|
|
36
|
-
|
37
|
-
|
56
|
+
def check_argument_size(argv, check_method, n)
|
57
|
+
unless argv.size.__send__(check_method, n)
|
58
|
+
raise "Invalid arguments number. Please refer '#{File.basename(__FILE__)} -h'."
|
59
|
+
end
|
38
60
|
end
|
39
61
|
|
40
|
-
|
41
|
-
uri = ARGV[1]
|
42
|
-
|
43
|
-
manage = DRbQS::Manage.new(uri)
|
62
|
+
manage = DRbQS::Manage.new
|
44
63
|
|
45
64
|
case command
|
46
65
|
when 'exit-signal'
|
47
|
-
|
66
|
+
check_argument_size(argv, :==, 2)
|
67
|
+
uri = argv[1]
|
68
|
+
manage.send_exit_signal(uri)
|
69
|
+
when 'initialize'
|
70
|
+
check_argument_size(argv, :==, 1)
|
71
|
+
manage.create_config
|
72
|
+
when 'ssh'
|
73
|
+
check_argument_size(argv, :>=, 2)
|
74
|
+
dest = argv[1]
|
75
|
+
manage.execute_over_ssh(dest, options, command_args)
|
48
76
|
else
|
49
77
|
raise "Invalid command: #{command}"
|
50
78
|
end
|
data/bin/drbqs-server
CHANGED
@@ -13,16 +13,12 @@ Start DRbQS server of definition files.
|
|
13
13
|
HELP
|
14
14
|
|
15
15
|
options = {
|
16
|
-
:log_file => STDOUT
|
16
|
+
:log_file => STDOUT,
|
17
|
+
:command_type => :server_start,
|
18
|
+
:acl => DRbQS::Config.get_acl_file
|
17
19
|
}
|
18
20
|
|
19
|
-
|
20
|
-
command_argv = ARGV[0..(n - 1)]
|
21
|
-
server_argv = ARGV[(n + 1)..-1]
|
22
|
-
else
|
23
|
-
command_argv = ARGV
|
24
|
-
server_argv = []
|
25
|
-
end
|
21
|
+
command_argv, server_argv = DRbQS::Manage.split_arguments(ARGV)
|
26
22
|
|
27
23
|
begin
|
28
24
|
OptionParser.new(help_message) do |opt|
|
@@ -46,6 +42,9 @@ begin
|
|
46
42
|
opt.on('--debug', 'Set $DEBUG true.') do |v|
|
47
43
|
$DEBUG = true
|
48
44
|
end
|
45
|
+
opt.on('--test STR', String, 'Execute test.') do |v|
|
46
|
+
options[:command_type] = "test_#{v}"
|
47
|
+
end
|
49
48
|
opt.parse!(command_argv)
|
50
49
|
end
|
51
50
|
rescue OptionParser::InvalidOption
|
@@ -72,4 +71,11 @@ command_argv.each do |path|
|
|
72
71
|
end
|
73
72
|
|
74
73
|
DRbQS.parse_option(server_argv)
|
75
|
-
|
74
|
+
case options[:command_type]
|
75
|
+
when /^test/
|
76
|
+
s = options[:command_type].split('_')[1].split(',')
|
77
|
+
type = s[0].intern
|
78
|
+
DRbQS.test_server(options, type, s[1..-1])
|
79
|
+
else
|
80
|
+
DRbQS.start_server(options)
|
81
|
+
end
|
data/drbqs.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{drbqs}
|
8
|
-
s.version = "0.0.
|
8
|
+
s.version = "0.0.8"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Takayuki YAMAGUCHI"]
|
12
|
-
s.date = %q{2011-03-
|
12
|
+
s.date = %q{2011-03-20}
|
13
13
|
s.description = %q{Task queuing system over network that is implemented by dRuby.}
|
14
14
|
s.email = %q{d@ytak.info}
|
15
15
|
s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server"]
|
@@ -30,15 +30,19 @@ Gem::Specification.new do |s|
|
|
30
30
|
"bin/drbqs-server",
|
31
31
|
"drbqs.gemspec",
|
32
32
|
"example/README.md",
|
33
|
+
"example/command/server_def.rb",
|
33
34
|
"example/drbqs-manage-test.rb",
|
34
35
|
"example/drbqs-node-test.rb",
|
35
36
|
"example/drbqs-server-test.rb",
|
36
|
-
"example/server/
|
37
|
+
"example/server/server.rb",
|
37
38
|
"example/sum/server_def.rb",
|
38
39
|
"example/sum/sum.rb",
|
40
|
+
"example/sum2/server_def.rb",
|
41
|
+
"example/sum2/sum.rb",
|
39
42
|
"lib/drbqs.rb",
|
40
43
|
"lib/drbqs/acl_file.rb",
|
41
44
|
"lib/drbqs/client.rb",
|
45
|
+
"lib/drbqs/config.rb",
|
42
46
|
"lib/drbqs/connection.rb",
|
43
47
|
"lib/drbqs/manage.rb",
|
44
48
|
"lib/drbqs/message.rb",
|
@@ -47,10 +51,12 @@ Gem::Specification.new do |s|
|
|
47
51
|
"lib/drbqs/server.rb",
|
48
52
|
"lib/drbqs/server_define.rb",
|
49
53
|
"lib/drbqs/server_hook.rb",
|
54
|
+
"lib/drbqs/ssh_shell.rb",
|
50
55
|
"lib/drbqs/task.rb",
|
51
56
|
"lib/drbqs/task_client.rb",
|
52
57
|
"lib/drbqs/task_generator.rb",
|
53
58
|
"spec/acl_file_spec.rb",
|
59
|
+
"spec/config_spec.rb",
|
54
60
|
"spec/connection_spec.rb",
|
55
61
|
"spec/data/acl.txt",
|
56
62
|
"spec/manage_spec.rb",
|
@@ -61,6 +67,7 @@ Gem::Specification.new do |s|
|
|
61
67
|
"spec/server_hook_spec.rb",
|
62
68
|
"spec/server_spec.rb",
|
63
69
|
"spec/spec_helper.rb",
|
70
|
+
"spec/ssh_shell_spec.rb",
|
64
71
|
"spec/task_client_spec.rb",
|
65
72
|
"spec/task_generator_spec.rb",
|
66
73
|
"spec/task_spec.rb",
|
@@ -71,10 +78,11 @@ Gem::Specification.new do |s|
|
|
71
78
|
s.homepage = %q{http://github.com/ytaka/drbqs}
|
72
79
|
s.licenses = ["GPL3"]
|
73
80
|
s.require_paths = ["lib"]
|
74
|
-
s.rubygems_version = %q{1.
|
81
|
+
s.rubygems_version = %q{1.5.2}
|
75
82
|
s.summary = %q{dRuby Queueing System}
|
76
83
|
s.test_files = [
|
77
84
|
"spec/acl_file_spec.rb",
|
85
|
+
"spec/config_spec.rb",
|
78
86
|
"spec/connection_spec.rb",
|
79
87
|
"spec/manage_spec.rb",
|
80
88
|
"spec/message_spec.rb",
|
@@ -84,6 +92,7 @@ Gem::Specification.new do |s|
|
|
84
92
|
"spec/server_hook_spec.rb",
|
85
93
|
"spec/server_spec.rb",
|
86
94
|
"spec/spec_helper.rb",
|
95
|
+
"spec/ssh_shell_spec.rb",
|
87
96
|
"spec/task_client_spec.rb",
|
88
97
|
"spec/task_generator_spec.rb",
|
89
98
|
"spec/task_spec.rb",
|
@@ -0,0 +1,14 @@
|
|
1
|
+
#
|
2
|
+
# Usage:
|
3
|
+
# drbqs-server server_def.rb
|
4
|
+
#
|
5
|
+
|
6
|
+
DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
|
7
|
+
tgen = DRbQS::TaskGenerator.new(:sleep_time => 2)
|
8
|
+
tgen.set do
|
9
|
+
3.times do |i|
|
10
|
+
add_task(DRbQS::CommandTask.new(["sleep #{@sleep_time}", 'echo hello world']))
|
11
|
+
end
|
12
|
+
end
|
13
|
+
server.add_task_generator(tgen)
|
14
|
+
end
|
data/example/sum/server_def.rb
CHANGED
@@ -12,12 +12,14 @@ DRbQS.option_parser do |opt, hash|
|
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
-
DRbQS.define_server do |server, argv, opts|
|
15
|
+
DRbQS.define_server(:check_alive => 5) do |server, argv, opts|
|
16
16
|
start_num = (argv[0] || 10).to_i
|
17
|
-
end_num = (argv[1] ||
|
17
|
+
end_num = (argv[1] || 50).to_i
|
18
18
|
step_num = opts[:step] || 10
|
19
19
|
start_num.step(end_num, step_num) do |i|
|
20
|
-
task = DRbQS::Task.new(Sum.new(i - 10, i), :exec)
|
20
|
+
task = DRbQS::Task.new(Sum.new(i - 10, i), :exec) do |srv, ret|
|
21
|
+
puts "Receive: #{ret.inspect}"
|
22
|
+
end
|
21
23
|
server.queue.add(task)
|
22
24
|
end
|
23
25
|
|
data/example/sum/sum.rb
CHANGED
@@ -0,0 +1,27 @@
|
|
1
|
+
#
|
2
|
+
# Usage:
|
3
|
+
# drbqs-server server_def.rb -- 30 50
|
4
|
+
# drbqs-server server_def.rb -- 100 500 --step 100
|
5
|
+
#
|
6
|
+
|
7
|
+
require_relative 'sum.rb'
|
8
|
+
|
9
|
+
DRbQS.option_parser do |opt, hash|
|
10
|
+
opt.on('--step NUM', Integer) do |v|
|
11
|
+
hash[:step] = v
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
|
16
|
+
tgen = DRbQS::TaskGenerator.new(:start_num => (argv[0] || 10).to_i,
|
17
|
+
:end_num => (argv[1] || 100).to_i,
|
18
|
+
:step_num => opts[:step] || 10)
|
19
|
+
tgen.set do
|
20
|
+
@start_num.step(@end_num, @step_num) do |i|
|
21
|
+
create_add_task(Sum.new(i - 10, i), :exec) do |srv, ret|
|
22
|
+
puts "Receive: #{ret.inspect}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
server.add_task_generator(tgen)
|
27
|
+
end
|
data/example/sum2/sum.rb
ADDED
data/lib/drbqs/config.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'singleton'
|
3
|
+
|
4
|
+
module DRbQS
|
5
|
+
|
6
|
+
class Config
|
7
|
+
|
8
|
+
@@data = {
|
9
|
+
:dir => ENV['HOME'] + '/.drbqs/'
|
10
|
+
}
|
11
|
+
|
12
|
+
ACL_SAMPLE =<<SAMPLE
|
13
|
+
deny all
|
14
|
+
allow localhost
|
15
|
+
allow 127.0.0.1
|
16
|
+
SAMPLE
|
17
|
+
|
18
|
+
class << self
|
19
|
+
def get_path(name)
|
20
|
+
File.join(@@data[:dir], name)
|
21
|
+
end
|
22
|
+
private :get_path
|
23
|
+
|
24
|
+
def set_directory(dir)
|
25
|
+
@@data[:dir] = dir
|
26
|
+
end
|
27
|
+
|
28
|
+
def check_directory_create
|
29
|
+
unless File.exist?(@@data[:dir])
|
30
|
+
FileUtils.mkdir_p(@@data[:dir])
|
31
|
+
FileUtils.chmod(0700, @@data[:dir])
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def save_sample
|
36
|
+
path = get_path('acl.txt.sample')
|
37
|
+
unless File.exist?(path)
|
38
|
+
open(path, 'w') { |f| f.print ACL_SAMPLE }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def get_acl_file
|
43
|
+
path = File.join(@@data[:dir], 'acl.txt')
|
44
|
+
if File.exist?(path)
|
45
|
+
return path
|
46
|
+
end
|
47
|
+
return nil
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
data/lib/drbqs/manage.rb
CHANGED
@@ -3,13 +3,28 @@ require 'socket'
|
|
3
3
|
module DRbQS
|
4
4
|
|
5
5
|
class Manage
|
6
|
-
def
|
7
|
-
|
6
|
+
def self.split_arguments(argv, split = '--')
|
7
|
+
if n = argv.index(split)
|
8
|
+
[argv[0..(n - 1)], argv[(n + 1)..-1]]
|
9
|
+
else
|
10
|
+
[argv, []]
|
11
|
+
end
|
8
12
|
end
|
9
13
|
|
10
|
-
def
|
11
|
-
|
14
|
+
def create_config
|
15
|
+
Config.check_directory_create
|
16
|
+
Config.save_sample
|
17
|
+
end
|
18
|
+
|
19
|
+
def send_exit_signal(access_uri)
|
20
|
+
obj = DRbObject.new_with_uri(access_uri)
|
12
21
|
obj[:message].write([:exit_server, "Command of #{Socket.gethostname}"])
|
13
22
|
end
|
23
|
+
|
24
|
+
def execute_over_ssh(dest, opts, command)
|
25
|
+
ssh = DRbQS::SSHShell.new(dest, opts)
|
26
|
+
ssh.get_environment
|
27
|
+
ssh.start(command)
|
28
|
+
end
|
14
29
|
end
|
15
30
|
end
|
data/lib/drbqs/queue.rb
CHANGED
@@ -44,7 +44,10 @@ module DRbQS
|
|
44
44
|
def requeue_for_deleted_node_id(deleted)
|
45
45
|
deleted.each do |node_id|
|
46
46
|
if task_id_ary = @calculating[node_id]
|
47
|
-
task_id_ary.each
|
47
|
+
task_id_ary.each do |task_id|
|
48
|
+
queue_task(task_id)
|
49
|
+
@logger.info("Requeue: task #{task_id}.") if @logger
|
50
|
+
end
|
48
51
|
@calculating.delete(node_id)
|
49
52
|
end
|
50
53
|
end
|
data/lib/drbqs/server.rb
CHANGED
@@ -5,8 +5,10 @@ require 'drbqs/server_hook'
|
|
5
5
|
|
6
6
|
module DRbQS
|
7
7
|
class CheckAlive
|
8
|
+
DEFAULT_INTERVAL_TIME = 300
|
9
|
+
|
8
10
|
def initialize(interval)
|
9
|
-
@interval = interval ||
|
11
|
+
@interval = interval || DEFAULT_INTERVAL_TIME
|
10
12
|
@last = Time.now
|
11
13
|
end
|
12
14
|
|
@@ -22,6 +24,9 @@ module DRbQS
|
|
22
24
|
# When we set both empty_queue_hook and task_generator,
|
23
25
|
# empty_queue_hook is prior to task_generator.
|
24
26
|
class Server
|
27
|
+
WAIT_TIME_NODE_EXIT = 3
|
28
|
+
WAIT_TIME_NEW_RESULT = 1
|
29
|
+
|
25
30
|
attr_reader :queue
|
26
31
|
|
27
32
|
# :port
|
@@ -88,7 +93,8 @@ module DRbQS
|
|
88
93
|
def check_connection(force = nil)
|
89
94
|
if force || @check_alive.significant_interval?
|
90
95
|
@logger.debug("Check connection") if @logger
|
91
|
-
@message.check_connection
|
96
|
+
deleted_node_ids = @message.check_connection
|
97
|
+
@queue.requeue_for_deleted_node_id(deleted_node_ids)
|
92
98
|
@check_alive.set_checking
|
93
99
|
end
|
94
100
|
end
|
@@ -102,7 +108,7 @@ module DRbQS
|
|
102
108
|
if @task_generator.size > 0 && @queue.empty?
|
103
109
|
if tasks = @task_generator[0].new_tasks
|
104
110
|
tasks.each { |t| @queue.add(t) }
|
105
|
-
@logger.
|
111
|
+
@logger.info("Generator add #{tasks.size} tasks.") if @logger
|
106
112
|
else
|
107
113
|
@task_generator.delete_at(0)
|
108
114
|
@logger.info("Generator creates all tasks and then has been deleted.") if @logger
|
@@ -141,13 +147,10 @@ module DRbQS
|
|
141
147
|
end
|
142
148
|
private :exec_hook
|
143
149
|
|
144
|
-
WAIT_NODE_EXIT = 3
|
145
|
-
WAIT_NEW_RESULT = 1
|
146
|
-
|
147
150
|
def exit
|
148
151
|
@message.send_exit
|
149
152
|
until @message.node_not_exist?
|
150
|
-
sleep(
|
153
|
+
sleep(WAIT_TIME_NODE_EXIT)
|
151
154
|
check_connection(true)
|
152
155
|
end
|
153
156
|
Kernel.exit
|
@@ -166,8 +169,13 @@ module DRbQS
|
|
166
169
|
end
|
167
170
|
private :check_message
|
168
171
|
|
169
|
-
def
|
172
|
+
def task_generator_init
|
170
173
|
@task_generator.each { |tgen| tgen.init }
|
174
|
+
end
|
175
|
+
private :task_generator_init
|
176
|
+
|
177
|
+
def wait
|
178
|
+
task_generator_init
|
171
179
|
loop do
|
172
180
|
check_message
|
173
181
|
check_connection
|
@@ -175,9 +183,18 @@ module DRbQS
|
|
175
183
|
exec_hook
|
176
184
|
@logger.debug("Calculating tasks: #{@queue.calculating_task_number}") if @logger
|
177
185
|
if count_results <= 1
|
178
|
-
sleep(
|
186
|
+
sleep(WAIT_TIME_NEW_RESULT)
|
179
187
|
end
|
180
188
|
end
|
181
189
|
end
|
190
|
+
|
191
|
+
def test_task_generator(opts = {})
|
192
|
+
task_generator_init
|
193
|
+
@task_generator.each_with_index do |t, i|
|
194
|
+
puts "Test task generator [#{i}]"
|
195
|
+
set_num, task_num = t.debug_all_tasks(opts)
|
196
|
+
puts "Create: task sets #{set_num}, all tasks #{task_num}"
|
197
|
+
end
|
198
|
+
end
|
182
199
|
end
|
183
200
|
end
|
data/lib/drbqs/server_define.rb
CHANGED
@@ -12,9 +12,11 @@ HELP
|
|
12
12
|
@option_parse = nil
|
13
13
|
@opts = {}
|
14
14
|
@argv = nil
|
15
|
+
@default_server_opts = nil
|
15
16
|
end
|
16
17
|
|
17
|
-
def define_server(&block)
|
18
|
+
def define_server(default_opts = {}, &block)
|
19
|
+
@default_server_opts = default_opts
|
18
20
|
if @server_create
|
19
21
|
raise ArgumentError, "The server has already defined."
|
20
22
|
end
|
@@ -38,33 +40,41 @@ HELP
|
|
38
40
|
@argv = opt_argv
|
39
41
|
end
|
40
42
|
|
43
|
+
def create_server(options)
|
44
|
+
server = DRbQS::Server.new(@default_server_opts.merge(options))
|
45
|
+
@server_create.call(server, @argv, @opts)
|
46
|
+
server.set_signal_trap
|
47
|
+
server
|
48
|
+
end
|
49
|
+
private :create_server
|
50
|
+
|
41
51
|
def start_server(options)
|
42
52
|
unless @server_create
|
43
53
|
raise "Can not get server definition."
|
44
54
|
end
|
45
|
-
server =
|
46
|
-
@server_create.call(server, @argv, @opts)
|
47
|
-
server.set_signal_trap
|
55
|
+
server = create_server(options)
|
48
56
|
server.start
|
49
57
|
server.wait
|
50
58
|
end
|
51
|
-
end
|
52
|
-
|
53
|
-
@@server_def = ServerDefinition.new
|
54
|
-
|
55
|
-
def self.define_server(&block)
|
56
|
-
@@server_def.define_server(&block)
|
57
|
-
end
|
58
59
|
|
59
|
-
|
60
|
-
|
60
|
+
def test_server(options, type, arg = [])
|
61
|
+
server = create_server(options)
|
62
|
+
case type
|
63
|
+
when :task
|
64
|
+
puts "*** Test of Task Generators ***"
|
65
|
+
server.test_task_generator(:limit => arg[0] ? arg[0].to_i : nil, :progress => true)
|
66
|
+
else
|
67
|
+
puts "*** Not be yet implemented ***"
|
68
|
+
end
|
69
|
+
end
|
61
70
|
end
|
62
71
|
|
63
|
-
|
64
|
-
@@server_def.parse_option(opt_argv)
|
65
|
-
end
|
72
|
+
@@server_def = ServerDefinition.new
|
66
73
|
|
67
|
-
|
68
|
-
|
74
|
+
class << self
|
75
|
+
[:define_server, :option_parser, :parse_option,
|
76
|
+
:start_server, :test_server].each do |m|
|
77
|
+
define_method(m, &@@server_def.method(m))
|
78
|
+
end
|
69
79
|
end
|
70
80
|
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'net/ssh'
|
2
|
+
require 'net/ssh/shell'
|
3
|
+
|
4
|
+
module DRbQS
|
5
|
+
|
6
|
+
# Requirements:
|
7
|
+
# bash
|
8
|
+
# nohup
|
9
|
+
class SSHShell
|
10
|
+
class GetInvalidExitStatus < StandardError
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :user, :host, :directory, :port
|
14
|
+
|
15
|
+
DEFAULT_RVM_SCRIPT = '$HOME/.rvm/scripts/rvm'
|
16
|
+
DEFAULT_OUTPUT_FILE = 'drbqs_nohup.log'
|
17
|
+
|
18
|
+
# :shell shell to use
|
19
|
+
# :dir base directory of ssh server
|
20
|
+
# :rvm version of ruby on rvm
|
21
|
+
# :rvm_init path of script to initialize rvm
|
22
|
+
# :output file to output of stdout and stderr
|
23
|
+
def initialize(dest, opts = {})
|
24
|
+
@user, @host, @port = split_destination(dest)
|
25
|
+
if !(@host && @user)
|
26
|
+
raise "Invalid destination of ssh server."
|
27
|
+
end
|
28
|
+
@shell = opts[:shell] || 'bash'
|
29
|
+
@rvm = opts[:rvm]
|
30
|
+
@rvm_init = opts[:rvm_init]
|
31
|
+
if (@rvm || @rvm_init) && !(String === @rvm_init)
|
32
|
+
@rvm_init = DEFAULT_RVM_SCRIPT
|
33
|
+
end
|
34
|
+
@out = opts[:output] || DEFAULT_OUTPUT_FILE
|
35
|
+
@directory = opts[:dir]
|
36
|
+
end
|
37
|
+
|
38
|
+
def split_destination(dest)
|
39
|
+
if (n = dest.index("@")) && n > 0
|
40
|
+
user = dest[0..(n - 1)]
|
41
|
+
host_dir = dest[(n + 1)..-1]
|
42
|
+
else
|
43
|
+
raise "Not include '@': #{dest}"
|
44
|
+
end
|
45
|
+
if n = host_dir.index(':')
|
46
|
+
host = host_dir[0..(n - 1)]
|
47
|
+
port = host_dir[(n + 1)..-1]
|
48
|
+
else
|
49
|
+
host = host_dir
|
50
|
+
port = nil
|
51
|
+
end
|
52
|
+
[user && user.size > 0 ? user : nil,
|
53
|
+
host && host.size > 0 ? host : nil,
|
54
|
+
port && port.size > 0 ? port.to_i : nil]
|
55
|
+
end
|
56
|
+
private :split_destination
|
57
|
+
|
58
|
+
def shell_exec(sh, cmd)
|
59
|
+
pr = sh.execute!(cmd)
|
60
|
+
if pr.exit_status != 0
|
61
|
+
raise GetInvalidExitStatus, "Can not execute '#{cmd}' on #{@host} properly."
|
62
|
+
end
|
63
|
+
end
|
64
|
+
private :shell_exec
|
65
|
+
|
66
|
+
def execute_command(*cmds)
|
67
|
+
Net::SSH.start(@host, @user, :port => @port) do |ssh|
|
68
|
+
ssh.shell(@shell) do |sh|
|
69
|
+
shell_exec(sh, "cd #{@directory}") if @directory
|
70
|
+
shell_exec(sh, "source #{@rvm_init}") if @rvm_init
|
71
|
+
shell_exec(sh, "rvm use #{@rvm}") if @rvm
|
72
|
+
cmds.each do |c|
|
73
|
+
sh.execute c
|
74
|
+
end
|
75
|
+
sh.execute "exit"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
private :execute_command
|
80
|
+
|
81
|
+
def get_environment
|
82
|
+
execute_command('echo "directory: " `pwd`',
|
83
|
+
'echo "files:"',
|
84
|
+
'ls',
|
85
|
+
'if which rvm > /dev/null; then rvm info; else ruby -v; fi')
|
86
|
+
end
|
87
|
+
|
88
|
+
def start(*args)
|
89
|
+
execute_command("nohup #{args.join(' ')} > #{@out} 2>&1 &")
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
data/lib/drbqs/task.rb
CHANGED
@@ -8,7 +8,7 @@ module DRbQS
|
|
8
8
|
attr_reader :hook
|
9
9
|
|
10
10
|
# Nodes execute obj.method_sym(*args).
|
11
|
-
# Server executes &hook with a server instance and an
|
11
|
+
# Server executes &hook with a server instance and an object of result
|
12
12
|
# after the server accepts the results from nodes.
|
13
13
|
def initialize(obj, method_sym, args = [], &hook)
|
14
14
|
begin
|
@@ -35,4 +35,29 @@ module DRbQS
|
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
38
|
+
class CommandExecute
|
39
|
+
def initialize(cmd)
|
40
|
+
@cmd = cmd
|
41
|
+
unless (Array === @cmd || String === @cmd)
|
42
|
+
raise ArgumentError, "Invalid command: #{@cmd.inspect}"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def exec
|
47
|
+
case @cmd
|
48
|
+
when Array
|
49
|
+
@cmd.each { |c| system(c) }
|
50
|
+
when String
|
51
|
+
system(@cmd)
|
52
|
+
end
|
53
|
+
$?.exitstatus
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
class CommandTask < Task
|
58
|
+
# &hook takes a server instance and exit number of command.
|
59
|
+
def initialize(cmd, &hook)
|
60
|
+
super(CommandExecute.new(cmd), :exec, &hook)
|
61
|
+
end
|
62
|
+
end
|
38
63
|
end
|
data/lib/drbqs/task_generator.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
module DRbQS
|
2
|
+
class DRbQS::TaskCreatingError < StandardError
|
3
|
+
end
|
4
|
+
|
2
5
|
class TaskGenerator
|
3
6
|
def initialize(data = {})
|
4
7
|
data.each do |key, val|
|
@@ -32,7 +35,11 @@ module DRbQS
|
|
32
35
|
@__iterate__ = iterate
|
33
36
|
@__fiber_init__ = lambda do
|
34
37
|
@__fiber__ = Fiber.new do
|
35
|
-
|
38
|
+
begin
|
39
|
+
instance_eval(&block)
|
40
|
+
rescue => err
|
41
|
+
raise DRbQS::TaskCreatingError, "\n #{err.to_s}"
|
42
|
+
end
|
36
43
|
nil
|
37
44
|
end
|
38
45
|
end
|
@@ -68,8 +75,12 @@ module DRbQS
|
|
68
75
|
nil
|
69
76
|
end
|
70
77
|
|
78
|
+
DEBUG_TASK_PROGRESS = 1000
|
79
|
+
|
71
80
|
# Create all tasks for test and return [group_number, task_number] if all tasks created properly.
|
72
|
-
def debug_all_tasks(
|
81
|
+
def debug_all_tasks(opts = {})
|
82
|
+
limit = opts[:limit]
|
83
|
+
progress = opts[:progress]
|
73
84
|
group_number = 0
|
74
85
|
task_number = 0
|
75
86
|
while ary = new_tasks
|
@@ -78,6 +89,12 @@ module DRbQS
|
|
78
89
|
raise "Invalid #{i}th task: #{t.inspect}"
|
79
90
|
end
|
80
91
|
task_number += 1
|
92
|
+
if progress && (task_number % DEBUG_TASK_PROGRESS == 0)
|
93
|
+
puts "#{task_number} tasks have been created."
|
94
|
+
end
|
95
|
+
if limit && task_number > limit
|
96
|
+
break
|
97
|
+
end
|
81
98
|
end
|
82
99
|
group_number += 1
|
83
100
|
end
|
data/lib/drbqs.rb
CHANGED
@@ -14,6 +14,10 @@ module DRbQS
|
|
14
14
|
autoload :Task, 'drbqs/task'
|
15
15
|
autoload :TaskGenerator, 'drbqs/task_generator'
|
16
16
|
autoload :Manage, 'drbqs/manage'
|
17
|
+
autoload :Config, 'drbqs/config'
|
18
|
+
autoload :SSHShell, 'drbqs/ssh_shell'
|
19
|
+
autoload :CommandTask, 'drbqs/task'
|
20
|
+
autoload :CommandExecute, 'drbqs/task'
|
17
21
|
|
18
22
|
ROOT_DEFAULT_PORT = 13500
|
19
23
|
end
|
data/spec/config_spec.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
describe DRbQS::Config do
|
4
|
+
it "should return nil" do
|
5
|
+
DRbQS::Config.set_directory(File.dirname(__FILE__) + '/not_exist_path')
|
6
|
+
DRbQS::Config.get_acl_file.should be_nil
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should return existing path" do
|
10
|
+
path = File.dirname(__FILE__) + '/data'
|
11
|
+
DRbQS::Config.set_directory(path)
|
12
|
+
DRbQS::Config.get_acl_file.should == File.join(path, 'acl.txt')
|
13
|
+
end
|
14
|
+
end
|
data/spec/manage_spec.rb
CHANGED
@@ -27,12 +27,12 @@ describe DRbQS do
|
|
27
27
|
|
28
28
|
@uri = 'druby://:13501'
|
29
29
|
|
30
|
-
@manage = DRbQS::Manage.new
|
30
|
+
@manage = DRbQS::Manage.new
|
31
31
|
end
|
32
32
|
|
33
33
|
it "should send exit signal" do
|
34
34
|
lambda do
|
35
|
-
@manage.send_exit_signal
|
35
|
+
@manage.send_exit_signal(@uri)
|
36
36
|
end.should_not raise_error
|
37
37
|
lambda do
|
38
38
|
i = 0
|
@@ -47,4 +47,11 @@ describe DRbQS do
|
|
47
47
|
end.should_not raise_error
|
48
48
|
end
|
49
49
|
|
50
|
+
it "should split arguments" do
|
51
|
+
ary = ['abc', 'def', '--', '123', '45', '6']
|
52
|
+
a1, a2 = DRbQS::Manage.split_arguments(ary)
|
53
|
+
a1.should == ['abc', 'def']
|
54
|
+
a2.should == ['123', '45', '6']
|
55
|
+
end
|
56
|
+
|
50
57
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
require 'drbqs/ssh_shell'
|
4
|
+
|
5
|
+
describe DRbQS::SSHShell do
|
6
|
+
it "should split destination" do
|
7
|
+
ssh = DRbQS::SSHShell.new('user@hostname')
|
8
|
+
ssh.user.should == 'user'
|
9
|
+
ssh.host.should == 'hostname'
|
10
|
+
ssh.port.should be_nil
|
11
|
+
ssh.directory.should be_nil
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should split destination including directory" do
|
15
|
+
ssh = DRbQS::SSHShell.new('user@hostname:22', :dir => '/path/to/directory')
|
16
|
+
ssh.user.should == 'user'
|
17
|
+
ssh.host.should == 'hostname'
|
18
|
+
ssh.port.should == 22
|
19
|
+
ssh.directory.should == '/path/to/directory'
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should raise error: not include '@'" do
|
23
|
+
lambda do
|
24
|
+
DRbQS::SSHShell.new('userhostname')
|
25
|
+
end.should raise_error
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should raise error: empty user name" do
|
29
|
+
lambda do
|
30
|
+
DRbQS::SSHShell.new('@hostname')
|
31
|
+
end.should raise_error
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should raise error: empty host name" do
|
35
|
+
lambda do
|
36
|
+
DRbQS::SSHShell.new('user:22')
|
37
|
+
end.should raise_error
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
metadata
CHANGED
@@ -1,83 +1,85 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: drbqs
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.7
|
3
|
+
version: !ruby/object:Gem::Version
|
5
4
|
prerelease:
|
5
|
+
version: 0.0.8
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Takayuki YAMAGUCHI
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
12
|
+
|
13
|
+
date: 2011-03-20 00:00:00 +09:00
|
13
14
|
default_executable:
|
14
|
-
dependencies:
|
15
|
-
- !ruby/object:Gem::Dependency
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
16
17
|
name: rspec
|
17
|
-
requirement: &
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
18
19
|
none: false
|
19
|
-
requirements:
|
20
|
-
- -
|
21
|
-
- !ruby/object:Gem::Version
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
22
23
|
version: 2.5.0
|
23
24
|
type: :development
|
24
25
|
prerelease: false
|
25
|
-
version_requirements: *
|
26
|
-
- !ruby/object:Gem::Dependency
|
26
|
+
version_requirements: *id001
|
27
|
+
- !ruby/object:Gem::Dependency
|
27
28
|
name: yard
|
28
|
-
requirement: &
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
29
30
|
none: false
|
30
|
-
requirements:
|
31
|
+
requirements:
|
31
32
|
- - ~>
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
+
- !ruby/object:Gem::Version
|
33
34
|
version: 0.6.0
|
34
35
|
type: :development
|
35
36
|
prerelease: false
|
36
|
-
version_requirements: *
|
37
|
-
- !ruby/object:Gem::Dependency
|
37
|
+
version_requirements: *id002
|
38
|
+
- !ruby/object:Gem::Dependency
|
38
39
|
name: bundler
|
39
|
-
requirement: &
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
40
41
|
none: false
|
41
|
-
requirements:
|
42
|
+
requirements:
|
42
43
|
- - ~>
|
43
|
-
- !ruby/object:Gem::Version
|
44
|
+
- !ruby/object:Gem::Version
|
44
45
|
version: 1.0.0
|
45
46
|
type: :development
|
46
47
|
prerelease: false
|
47
|
-
version_requirements: *
|
48
|
-
- !ruby/object:Gem::Dependency
|
48
|
+
version_requirements: *id003
|
49
|
+
- !ruby/object:Gem::Dependency
|
49
50
|
name: jeweler
|
50
|
-
requirement: &
|
51
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
51
52
|
none: false
|
52
|
-
requirements:
|
53
|
+
requirements:
|
53
54
|
- - ~>
|
54
|
-
- !ruby/object:Gem::Version
|
55
|
+
- !ruby/object:Gem::Version
|
55
56
|
version: 1.5.2
|
56
57
|
type: :development
|
57
58
|
prerelease: false
|
58
|
-
version_requirements: *
|
59
|
-
- !ruby/object:Gem::Dependency
|
59
|
+
version_requirements: *id004
|
60
|
+
- !ruby/object:Gem::Dependency
|
60
61
|
name: rcov
|
61
|
-
requirement: &
|
62
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
62
63
|
none: false
|
63
|
-
requirements:
|
64
|
-
- -
|
65
|
-
- !ruby/object:Gem::Version
|
66
|
-
version:
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: "0"
|
67
68
|
type: :development
|
68
69
|
prerelease: false
|
69
|
-
version_requirements: *
|
70
|
+
version_requirements: *id005
|
70
71
|
description: Task queuing system over network that is implemented by dRuby.
|
71
72
|
email: d@ytak.info
|
72
|
-
executables:
|
73
|
+
executables:
|
73
74
|
- drbqs-manage
|
74
75
|
- drbqs-node
|
75
76
|
- drbqs-server
|
76
77
|
extensions: []
|
77
|
-
|
78
|
+
|
79
|
+
extra_rdoc_files:
|
78
80
|
- LICENSE.txt
|
79
81
|
- README.rdoc
|
80
|
-
files:
|
82
|
+
files:
|
81
83
|
- .document
|
82
84
|
- .rspec
|
83
85
|
- Gemfile
|
@@ -90,15 +92,19 @@ files:
|
|
90
92
|
- bin/drbqs-server
|
91
93
|
- drbqs.gemspec
|
92
94
|
- example/README.md
|
95
|
+
- example/command/server_def.rb
|
93
96
|
- example/drbqs-manage-test.rb
|
94
97
|
- example/drbqs-node-test.rb
|
95
98
|
- example/drbqs-server-test.rb
|
96
|
-
- example/server/
|
99
|
+
- example/server/server.rb
|
97
100
|
- example/sum/server_def.rb
|
98
101
|
- example/sum/sum.rb
|
102
|
+
- example/sum2/server_def.rb
|
103
|
+
- example/sum2/sum.rb
|
99
104
|
- lib/drbqs.rb
|
100
105
|
- lib/drbqs/acl_file.rb
|
101
106
|
- lib/drbqs/client.rb
|
107
|
+
- lib/drbqs/config.rb
|
102
108
|
- lib/drbqs/connection.rb
|
103
109
|
- lib/drbqs/manage.rb
|
104
110
|
- lib/drbqs/message.rb
|
@@ -107,10 +113,12 @@ files:
|
|
107
113
|
- lib/drbqs/server.rb
|
108
114
|
- lib/drbqs/server_define.rb
|
109
115
|
- lib/drbqs/server_hook.rb
|
116
|
+
- lib/drbqs/ssh_shell.rb
|
110
117
|
- lib/drbqs/task.rb
|
111
118
|
- lib/drbqs/task_client.rb
|
112
119
|
- lib/drbqs/task_generator.rb
|
113
120
|
- spec/acl_file_spec.rb
|
121
|
+
- spec/config_spec.rb
|
114
122
|
- spec/connection_spec.rb
|
115
123
|
- spec/data/acl.txt
|
116
124
|
- spec/manage_spec.rb
|
@@ -121,6 +129,7 @@ files:
|
|
121
129
|
- spec/server_hook_spec.rb
|
122
130
|
- spec/server_spec.rb
|
123
131
|
- spec/spec_helper.rb
|
132
|
+
- spec/ssh_shell_spec.rb
|
124
133
|
- spec/task_client_spec.rb
|
125
134
|
- spec/task_generator_spec.rb
|
126
135
|
- spec/task_spec.rb
|
@@ -129,35 +138,38 @@ files:
|
|
129
138
|
- spec/test2_spec.rb
|
130
139
|
has_rdoc: true
|
131
140
|
homepage: http://github.com/ytaka/drbqs
|
132
|
-
licenses:
|
141
|
+
licenses:
|
133
142
|
- GPL3
|
134
143
|
post_install_message:
|
135
144
|
rdoc_options: []
|
136
|
-
|
145
|
+
|
146
|
+
require_paths:
|
137
147
|
- lib
|
138
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
148
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
139
149
|
none: false
|
140
|
-
requirements:
|
141
|
-
- -
|
142
|
-
- !ruby/object:Gem::Version
|
143
|
-
|
144
|
-
segments:
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
hash: 3292588984449245047
|
154
|
+
segments:
|
145
155
|
- 0
|
146
|
-
|
147
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
156
|
+
version: "0"
|
157
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
148
158
|
none: false
|
149
|
-
requirements:
|
150
|
-
- -
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version:
|
159
|
+
requirements:
|
160
|
+
- - ">="
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: "0"
|
153
163
|
requirements: []
|
164
|
+
|
154
165
|
rubyforge_project:
|
155
|
-
rubygems_version: 1.
|
166
|
+
rubygems_version: 1.5.2
|
156
167
|
signing_key:
|
157
168
|
specification_version: 3
|
158
169
|
summary: dRuby Queueing System
|
159
|
-
test_files:
|
170
|
+
test_files:
|
160
171
|
- spec/acl_file_spec.rb
|
172
|
+
- spec/config_spec.rb
|
161
173
|
- spec/connection_spec.rb
|
162
174
|
- spec/manage_spec.rb
|
163
175
|
- spec/message_spec.rb
|
@@ -167,6 +179,7 @@ test_files:
|
|
167
179
|
- spec/server_hook_spec.rb
|
168
180
|
- spec/server_spec.rb
|
169
181
|
- spec/spec_helper.rb
|
182
|
+
- spec/ssh_shell_spec.rb
|
170
183
|
- spec/task_client_spec.rb
|
171
184
|
- spec/task_generator_spec.rb
|
172
185
|
- spec/task_spec.rb
|
File without changes
|