drbqs 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/bin/drbqs-manage +38 -10
- data/bin/drbqs-server +15 -9
- data/drbqs.gemspec +13 -4
- data/example/command/server_def.rb +14 -0
- data/example/sum/server_def.rb +5 -3
- data/example/sum/sum.rb +1 -0
- data/example/sum2/server_def.rb +27 -0
- data/example/sum2/sum.rb +9 -0
- data/lib/drbqs/config.rb +52 -0
- data/lib/drbqs/manage.rb +19 -4
- data/lib/drbqs/queue.rb +4 -1
- data/lib/drbqs/server.rb +26 -9
- data/lib/drbqs/server_define.rb +28 -18
- data/lib/drbqs/ssh_shell.rb +92 -0
- data/lib/drbqs/task.rb +26 -1
- data/lib/drbqs/task_generator.rb +19 -2
- data/lib/drbqs.rb +4 -0
- data/spec/config_spec.rb +14 -0
- data/spec/manage_spec.rb +9 -2
- data/spec/ssh_shell_spec.rb +40 -0
- metadata +67 -54
- /data/example/server/{server_def.rb → server.rb} +0 -0
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.8
|
data/bin/drbqs-manage
CHANGED
@@ -6,18 +6,38 @@ require 'optparse'
|
|
6
6
|
Version = '0.0.1'
|
7
7
|
|
8
8
|
help_message =<<HELP
|
9
|
-
Usage: #{File.basename(__FILE__)} <command>
|
9
|
+
Usage: #{File.basename(__FILE__)} <command> [arguments ...]
|
10
10
|
Manage DRbQS server by sending messages.
|
11
|
-
<command> is 'exit-signal'.
|
11
|
+
<command> is 'exit-signal', 'initialize', or 'ssh'.
|
12
12
|
|
13
13
|
HELP
|
14
14
|
|
15
|
+
command = ARGV[0]
|
16
|
+
argv, command_args = DRbQS::Manage.split_arguments(ARGV)
|
17
|
+
|
18
|
+
options = {}
|
19
|
+
|
15
20
|
begin
|
16
21
|
OptionParser.new(help_message) do |opt|
|
17
22
|
opt.on('--debug', 'Set $DEBUG true.') do |v|
|
18
23
|
$DEBUG = true
|
19
24
|
end
|
20
|
-
opt.
|
25
|
+
opt.on('--dir DIR', String, 'Set the base directory over ssh.') do |v|
|
26
|
+
options[:dir] = v
|
27
|
+
end
|
28
|
+
opt.on('--shell STR', String, 'Set the shell over ssh') do |v|
|
29
|
+
options[:shell] = v
|
30
|
+
end
|
31
|
+
opt.on('--rvm STR', String, 'Ruby version to use on RVM over ssh.') do |v|
|
32
|
+
options[:rvm] = v
|
33
|
+
end
|
34
|
+
opt.on('--rvm-init PATH', String, 'Path of script to initialize RVM over ssh.') do |v|
|
35
|
+
options[:rvm_init] = v
|
36
|
+
end
|
37
|
+
opt.on('--output PATH', String, 'File path that stdout and stderr are output to over ssh.') do |v|
|
38
|
+
options[:output] = v
|
39
|
+
end
|
40
|
+
opt.parse!(argv)
|
21
41
|
end
|
22
42
|
rescue OptionParser::InvalidOption
|
23
43
|
$stderr.print <<MES
|
@@ -33,18 +53,26 @@ MES
|
|
33
53
|
exit(2)
|
34
54
|
end
|
35
55
|
|
36
|
-
|
37
|
-
|
56
|
+
def check_argument_size(argv, check_method, n)
|
57
|
+
unless argv.size.__send__(check_method, n)
|
58
|
+
raise "Invalid arguments number. Please refer '#{File.basename(__FILE__)} -h'."
|
59
|
+
end
|
38
60
|
end
|
39
61
|
|
40
|
-
|
41
|
-
uri = ARGV[1]
|
42
|
-
|
43
|
-
manage = DRbQS::Manage.new(uri)
|
62
|
+
manage = DRbQS::Manage.new
|
44
63
|
|
45
64
|
case command
|
46
65
|
when 'exit-signal'
|
47
|
-
|
66
|
+
check_argument_size(argv, :==, 2)
|
67
|
+
uri = argv[1]
|
68
|
+
manage.send_exit_signal(uri)
|
69
|
+
when 'initialize'
|
70
|
+
check_argument_size(argv, :==, 1)
|
71
|
+
manage.create_config
|
72
|
+
when 'ssh'
|
73
|
+
check_argument_size(argv, :>=, 2)
|
74
|
+
dest = argv[1]
|
75
|
+
manage.execute_over_ssh(dest, options, command_args)
|
48
76
|
else
|
49
77
|
raise "Invalid command: #{command}"
|
50
78
|
end
|
data/bin/drbqs-server
CHANGED
@@ -13,16 +13,12 @@ Start DRbQS server of definition files.
|
|
13
13
|
HELP
|
14
14
|
|
15
15
|
options = {
|
16
|
-
:log_file => STDOUT
|
16
|
+
:log_file => STDOUT,
|
17
|
+
:command_type => :server_start,
|
18
|
+
:acl => DRbQS::Config.get_acl_file
|
17
19
|
}
|
18
20
|
|
19
|
-
|
20
|
-
command_argv = ARGV[0..(n - 1)]
|
21
|
-
server_argv = ARGV[(n + 1)..-1]
|
22
|
-
else
|
23
|
-
command_argv = ARGV
|
24
|
-
server_argv = []
|
25
|
-
end
|
21
|
+
command_argv, server_argv = DRbQS::Manage.split_arguments(ARGV)
|
26
22
|
|
27
23
|
begin
|
28
24
|
OptionParser.new(help_message) do |opt|
|
@@ -46,6 +42,9 @@ begin
|
|
46
42
|
opt.on('--debug', 'Set $DEBUG true.') do |v|
|
47
43
|
$DEBUG = true
|
48
44
|
end
|
45
|
+
opt.on('--test STR', String, 'Execute test.') do |v|
|
46
|
+
options[:command_type] = "test_#{v}"
|
47
|
+
end
|
49
48
|
opt.parse!(command_argv)
|
50
49
|
end
|
51
50
|
rescue OptionParser::InvalidOption
|
@@ -72,4 +71,11 @@ command_argv.each do |path|
|
|
72
71
|
end
|
73
72
|
|
74
73
|
DRbQS.parse_option(server_argv)
|
75
|
-
|
74
|
+
case options[:command_type]
|
75
|
+
when /^test/
|
76
|
+
s = options[:command_type].split('_')[1].split(',')
|
77
|
+
type = s[0].intern
|
78
|
+
DRbQS.test_server(options, type, s[1..-1])
|
79
|
+
else
|
80
|
+
DRbQS.start_server(options)
|
81
|
+
end
|
data/drbqs.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{drbqs}
|
8
|
-
s.version = "0.0.
|
8
|
+
s.version = "0.0.8"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Takayuki YAMAGUCHI"]
|
12
|
-
s.date = %q{2011-03-
|
12
|
+
s.date = %q{2011-03-20}
|
13
13
|
s.description = %q{Task queuing system over network that is implemented by dRuby.}
|
14
14
|
s.email = %q{d@ytak.info}
|
15
15
|
s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server"]
|
@@ -30,15 +30,19 @@ Gem::Specification.new do |s|
|
|
30
30
|
"bin/drbqs-server",
|
31
31
|
"drbqs.gemspec",
|
32
32
|
"example/README.md",
|
33
|
+
"example/command/server_def.rb",
|
33
34
|
"example/drbqs-manage-test.rb",
|
34
35
|
"example/drbqs-node-test.rb",
|
35
36
|
"example/drbqs-server-test.rb",
|
36
|
-
"example/server/
|
37
|
+
"example/server/server.rb",
|
37
38
|
"example/sum/server_def.rb",
|
38
39
|
"example/sum/sum.rb",
|
40
|
+
"example/sum2/server_def.rb",
|
41
|
+
"example/sum2/sum.rb",
|
39
42
|
"lib/drbqs.rb",
|
40
43
|
"lib/drbqs/acl_file.rb",
|
41
44
|
"lib/drbqs/client.rb",
|
45
|
+
"lib/drbqs/config.rb",
|
42
46
|
"lib/drbqs/connection.rb",
|
43
47
|
"lib/drbqs/manage.rb",
|
44
48
|
"lib/drbqs/message.rb",
|
@@ -47,10 +51,12 @@ Gem::Specification.new do |s|
|
|
47
51
|
"lib/drbqs/server.rb",
|
48
52
|
"lib/drbqs/server_define.rb",
|
49
53
|
"lib/drbqs/server_hook.rb",
|
54
|
+
"lib/drbqs/ssh_shell.rb",
|
50
55
|
"lib/drbqs/task.rb",
|
51
56
|
"lib/drbqs/task_client.rb",
|
52
57
|
"lib/drbqs/task_generator.rb",
|
53
58
|
"spec/acl_file_spec.rb",
|
59
|
+
"spec/config_spec.rb",
|
54
60
|
"spec/connection_spec.rb",
|
55
61
|
"spec/data/acl.txt",
|
56
62
|
"spec/manage_spec.rb",
|
@@ -61,6 +67,7 @@ Gem::Specification.new do |s|
|
|
61
67
|
"spec/server_hook_spec.rb",
|
62
68
|
"spec/server_spec.rb",
|
63
69
|
"spec/spec_helper.rb",
|
70
|
+
"spec/ssh_shell_spec.rb",
|
64
71
|
"spec/task_client_spec.rb",
|
65
72
|
"spec/task_generator_spec.rb",
|
66
73
|
"spec/task_spec.rb",
|
@@ -71,10 +78,11 @@ Gem::Specification.new do |s|
|
|
71
78
|
s.homepage = %q{http://github.com/ytaka/drbqs}
|
72
79
|
s.licenses = ["GPL3"]
|
73
80
|
s.require_paths = ["lib"]
|
74
|
-
s.rubygems_version = %q{1.
|
81
|
+
s.rubygems_version = %q{1.5.2}
|
75
82
|
s.summary = %q{dRuby Queueing System}
|
76
83
|
s.test_files = [
|
77
84
|
"spec/acl_file_spec.rb",
|
85
|
+
"spec/config_spec.rb",
|
78
86
|
"spec/connection_spec.rb",
|
79
87
|
"spec/manage_spec.rb",
|
80
88
|
"spec/message_spec.rb",
|
@@ -84,6 +92,7 @@ Gem::Specification.new do |s|
|
|
84
92
|
"spec/server_hook_spec.rb",
|
85
93
|
"spec/server_spec.rb",
|
86
94
|
"spec/spec_helper.rb",
|
95
|
+
"spec/ssh_shell_spec.rb",
|
87
96
|
"spec/task_client_spec.rb",
|
88
97
|
"spec/task_generator_spec.rb",
|
89
98
|
"spec/task_spec.rb",
|
@@ -0,0 +1,14 @@
|
|
1
|
+
#
|
2
|
+
# Usage:
|
3
|
+
# drbqs-server server_def.rb
|
4
|
+
#
|
5
|
+
|
6
|
+
DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
|
7
|
+
tgen = DRbQS::TaskGenerator.new(:sleep_time => 2)
|
8
|
+
tgen.set do
|
9
|
+
3.times do |i|
|
10
|
+
add_task(DRbQS::CommandTask.new(["sleep #{@sleep_time}", 'echo hello world']))
|
11
|
+
end
|
12
|
+
end
|
13
|
+
server.add_task_generator(tgen)
|
14
|
+
end
|
data/example/sum/server_def.rb
CHANGED
@@ -12,12 +12,14 @@ DRbQS.option_parser do |opt, hash|
|
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
-
DRbQS.define_server do |server, argv, opts|
|
15
|
+
DRbQS.define_server(:check_alive => 5) do |server, argv, opts|
|
16
16
|
start_num = (argv[0] || 10).to_i
|
17
|
-
end_num = (argv[1] ||
|
17
|
+
end_num = (argv[1] || 50).to_i
|
18
18
|
step_num = opts[:step] || 10
|
19
19
|
start_num.step(end_num, step_num) do |i|
|
20
|
-
task = DRbQS::Task.new(Sum.new(i - 10, i), :exec)
|
20
|
+
task = DRbQS::Task.new(Sum.new(i - 10, i), :exec) do |srv, ret|
|
21
|
+
puts "Receive: #{ret.inspect}"
|
22
|
+
end
|
21
23
|
server.queue.add(task)
|
22
24
|
end
|
23
25
|
|
data/example/sum/sum.rb
CHANGED
@@ -0,0 +1,27 @@
|
|
1
|
+
#
|
2
|
+
# Usage:
|
3
|
+
# drbqs-server server_def.rb -- 30 50
|
4
|
+
# drbqs-server server_def.rb -- 100 500 --step 100
|
5
|
+
#
|
6
|
+
|
7
|
+
require_relative 'sum.rb'
|
8
|
+
|
9
|
+
DRbQS.option_parser do |opt, hash|
|
10
|
+
opt.on('--step NUM', Integer) do |v|
|
11
|
+
hash[:step] = v
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
|
16
|
+
tgen = DRbQS::TaskGenerator.new(:start_num => (argv[0] || 10).to_i,
|
17
|
+
:end_num => (argv[1] || 100).to_i,
|
18
|
+
:step_num => opts[:step] || 10)
|
19
|
+
tgen.set do
|
20
|
+
@start_num.step(@end_num, @step_num) do |i|
|
21
|
+
create_add_task(Sum.new(i - 10, i), :exec) do |srv, ret|
|
22
|
+
puts "Receive: #{ret.inspect}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
server.add_task_generator(tgen)
|
27
|
+
end
|
data/example/sum2/sum.rb
ADDED
data/lib/drbqs/config.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'singleton'
|
3
|
+
|
4
|
+
module DRbQS
|
5
|
+
|
6
|
+
class Config
|
7
|
+
|
8
|
+
@@data = {
|
9
|
+
:dir => ENV['HOME'] + '/.drbqs/'
|
10
|
+
}
|
11
|
+
|
12
|
+
ACL_SAMPLE =<<SAMPLE
|
13
|
+
deny all
|
14
|
+
allow localhost
|
15
|
+
allow 127.0.0.1
|
16
|
+
SAMPLE
|
17
|
+
|
18
|
+
class << self
|
19
|
+
def get_path(name)
|
20
|
+
File.join(@@data[:dir], name)
|
21
|
+
end
|
22
|
+
private :get_path
|
23
|
+
|
24
|
+
def set_directory(dir)
|
25
|
+
@@data[:dir] = dir
|
26
|
+
end
|
27
|
+
|
28
|
+
def check_directory_create
|
29
|
+
unless File.exist?(@@data[:dir])
|
30
|
+
FileUtils.mkdir_p(@@data[:dir])
|
31
|
+
FileUtils.chmod(0700, @@data[:dir])
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def save_sample
|
36
|
+
path = get_path('acl.txt.sample')
|
37
|
+
unless File.exist?(path)
|
38
|
+
open(path, 'w') { |f| f.print ACL_SAMPLE }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def get_acl_file
|
43
|
+
path = File.join(@@data[:dir], 'acl.txt')
|
44
|
+
if File.exist?(path)
|
45
|
+
return path
|
46
|
+
end
|
47
|
+
return nil
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
data/lib/drbqs/manage.rb
CHANGED
@@ -3,13 +3,28 @@ require 'socket'
|
|
3
3
|
module DRbQS
|
4
4
|
|
5
5
|
class Manage
|
6
|
-
def
|
7
|
-
|
6
|
+
def self.split_arguments(argv, split = '--')
|
7
|
+
if n = argv.index(split)
|
8
|
+
[argv[0..(n - 1)], argv[(n + 1)..-1]]
|
9
|
+
else
|
10
|
+
[argv, []]
|
11
|
+
end
|
8
12
|
end
|
9
13
|
|
10
|
-
def
|
11
|
-
|
14
|
+
def create_config
|
15
|
+
Config.check_directory_create
|
16
|
+
Config.save_sample
|
17
|
+
end
|
18
|
+
|
19
|
+
def send_exit_signal(access_uri)
|
20
|
+
obj = DRbObject.new_with_uri(access_uri)
|
12
21
|
obj[:message].write([:exit_server, "Command of #{Socket.gethostname}"])
|
13
22
|
end
|
23
|
+
|
24
|
+
def execute_over_ssh(dest, opts, command)
|
25
|
+
ssh = DRbQS::SSHShell.new(dest, opts)
|
26
|
+
ssh.get_environment
|
27
|
+
ssh.start(command)
|
28
|
+
end
|
14
29
|
end
|
15
30
|
end
|
data/lib/drbqs/queue.rb
CHANGED
@@ -44,7 +44,10 @@ module DRbQS
|
|
44
44
|
def requeue_for_deleted_node_id(deleted)
|
45
45
|
deleted.each do |node_id|
|
46
46
|
if task_id_ary = @calculating[node_id]
|
47
|
-
task_id_ary.each
|
47
|
+
task_id_ary.each do |task_id|
|
48
|
+
queue_task(task_id)
|
49
|
+
@logger.info("Requeue: task #{task_id}.") if @logger
|
50
|
+
end
|
48
51
|
@calculating.delete(node_id)
|
49
52
|
end
|
50
53
|
end
|
data/lib/drbqs/server.rb
CHANGED
@@ -5,8 +5,10 @@ require 'drbqs/server_hook'
|
|
5
5
|
|
6
6
|
module DRbQS
|
7
7
|
class CheckAlive
|
8
|
+
DEFAULT_INTERVAL_TIME = 300
|
9
|
+
|
8
10
|
def initialize(interval)
|
9
|
-
@interval = interval ||
|
11
|
+
@interval = interval || DEFAULT_INTERVAL_TIME
|
10
12
|
@last = Time.now
|
11
13
|
end
|
12
14
|
|
@@ -22,6 +24,9 @@ module DRbQS
|
|
22
24
|
# When we set both empty_queue_hook and task_generator,
|
23
25
|
# empty_queue_hook is prior to task_generator.
|
24
26
|
class Server
|
27
|
+
WAIT_TIME_NODE_EXIT = 3
|
28
|
+
WAIT_TIME_NEW_RESULT = 1
|
29
|
+
|
25
30
|
attr_reader :queue
|
26
31
|
|
27
32
|
# :port
|
@@ -88,7 +93,8 @@ module DRbQS
|
|
88
93
|
def check_connection(force = nil)
|
89
94
|
if force || @check_alive.significant_interval?
|
90
95
|
@logger.debug("Check connection") if @logger
|
91
|
-
@message.check_connection
|
96
|
+
deleted_node_ids = @message.check_connection
|
97
|
+
@queue.requeue_for_deleted_node_id(deleted_node_ids)
|
92
98
|
@check_alive.set_checking
|
93
99
|
end
|
94
100
|
end
|
@@ -102,7 +108,7 @@ module DRbQS
|
|
102
108
|
if @task_generator.size > 0 && @queue.empty?
|
103
109
|
if tasks = @task_generator[0].new_tasks
|
104
110
|
tasks.each { |t| @queue.add(t) }
|
105
|
-
@logger.
|
111
|
+
@logger.info("Generator add #{tasks.size} tasks.") if @logger
|
106
112
|
else
|
107
113
|
@task_generator.delete_at(0)
|
108
114
|
@logger.info("Generator creates all tasks and then has been deleted.") if @logger
|
@@ -141,13 +147,10 @@ module DRbQS
|
|
141
147
|
end
|
142
148
|
private :exec_hook
|
143
149
|
|
144
|
-
WAIT_NODE_EXIT = 3
|
145
|
-
WAIT_NEW_RESULT = 1
|
146
|
-
|
147
150
|
def exit
|
148
151
|
@message.send_exit
|
149
152
|
until @message.node_not_exist?
|
150
|
-
sleep(
|
153
|
+
sleep(WAIT_TIME_NODE_EXIT)
|
151
154
|
check_connection(true)
|
152
155
|
end
|
153
156
|
Kernel.exit
|
@@ -166,8 +169,13 @@ module DRbQS
|
|
166
169
|
end
|
167
170
|
private :check_message
|
168
171
|
|
169
|
-
def
|
172
|
+
def task_generator_init
|
170
173
|
@task_generator.each { |tgen| tgen.init }
|
174
|
+
end
|
175
|
+
private :task_generator_init
|
176
|
+
|
177
|
+
def wait
|
178
|
+
task_generator_init
|
171
179
|
loop do
|
172
180
|
check_message
|
173
181
|
check_connection
|
@@ -175,9 +183,18 @@ module DRbQS
|
|
175
183
|
exec_hook
|
176
184
|
@logger.debug("Calculating tasks: #{@queue.calculating_task_number}") if @logger
|
177
185
|
if count_results <= 1
|
178
|
-
sleep(
|
186
|
+
sleep(WAIT_TIME_NEW_RESULT)
|
179
187
|
end
|
180
188
|
end
|
181
189
|
end
|
190
|
+
|
191
|
+
def test_task_generator(opts = {})
|
192
|
+
task_generator_init
|
193
|
+
@task_generator.each_with_index do |t, i|
|
194
|
+
puts "Test task generator [#{i}]"
|
195
|
+
set_num, task_num = t.debug_all_tasks(opts)
|
196
|
+
puts "Create: task sets #{set_num}, all tasks #{task_num}"
|
197
|
+
end
|
198
|
+
end
|
182
199
|
end
|
183
200
|
end
|
data/lib/drbqs/server_define.rb
CHANGED
@@ -12,9 +12,11 @@ HELP
|
|
12
12
|
@option_parse = nil
|
13
13
|
@opts = {}
|
14
14
|
@argv = nil
|
15
|
+
@default_server_opts = nil
|
15
16
|
end
|
16
17
|
|
17
|
-
def define_server(&block)
|
18
|
+
def define_server(default_opts = {}, &block)
|
19
|
+
@default_server_opts = default_opts
|
18
20
|
if @server_create
|
19
21
|
raise ArgumentError, "The server has already defined."
|
20
22
|
end
|
@@ -38,33 +40,41 @@ HELP
|
|
38
40
|
@argv = opt_argv
|
39
41
|
end
|
40
42
|
|
43
|
+
def create_server(options)
|
44
|
+
server = DRbQS::Server.new(@default_server_opts.merge(options))
|
45
|
+
@server_create.call(server, @argv, @opts)
|
46
|
+
server.set_signal_trap
|
47
|
+
server
|
48
|
+
end
|
49
|
+
private :create_server
|
50
|
+
|
41
51
|
def start_server(options)
|
42
52
|
unless @server_create
|
43
53
|
raise "Can not get server definition."
|
44
54
|
end
|
45
|
-
server =
|
46
|
-
@server_create.call(server, @argv, @opts)
|
47
|
-
server.set_signal_trap
|
55
|
+
server = create_server(options)
|
48
56
|
server.start
|
49
57
|
server.wait
|
50
58
|
end
|
51
|
-
end
|
52
|
-
|
53
|
-
@@server_def = ServerDefinition.new
|
54
|
-
|
55
|
-
def self.define_server(&block)
|
56
|
-
@@server_def.define_server(&block)
|
57
|
-
end
|
58
59
|
|
59
|
-
|
60
|
-
|
60
|
+
def test_server(options, type, arg = [])
|
61
|
+
server = create_server(options)
|
62
|
+
case type
|
63
|
+
when :task
|
64
|
+
puts "*** Test of Task Generators ***"
|
65
|
+
server.test_task_generator(:limit => arg[0] ? arg[0].to_i : nil, :progress => true)
|
66
|
+
else
|
67
|
+
puts "*** Not be yet implemented ***"
|
68
|
+
end
|
69
|
+
end
|
61
70
|
end
|
62
71
|
|
63
|
-
|
64
|
-
@@server_def.parse_option(opt_argv)
|
65
|
-
end
|
72
|
+
@@server_def = ServerDefinition.new
|
66
73
|
|
67
|
-
|
68
|
-
|
74
|
+
class << self
|
75
|
+
[:define_server, :option_parser, :parse_option,
|
76
|
+
:start_server, :test_server].each do |m|
|
77
|
+
define_method(m, &@@server_def.method(m))
|
78
|
+
end
|
69
79
|
end
|
70
80
|
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'net/ssh'
|
2
|
+
require 'net/ssh/shell'
|
3
|
+
|
4
|
+
module DRbQS
|
5
|
+
|
6
|
+
# Requirements:
|
7
|
+
# bash
|
8
|
+
# nohup
|
9
|
+
class SSHShell
|
10
|
+
class GetInvalidExitStatus < StandardError
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :user, :host, :directory, :port
|
14
|
+
|
15
|
+
DEFAULT_RVM_SCRIPT = '$HOME/.rvm/scripts/rvm'
|
16
|
+
DEFAULT_OUTPUT_FILE = 'drbqs_nohup.log'
|
17
|
+
|
18
|
+
# :shell shell to use
|
19
|
+
# :dir base directory of ssh server
|
20
|
+
# :rvm version of ruby on rvm
|
21
|
+
# :rvm_init path of script to initialize rvm
|
22
|
+
# :output file to output of stdout and stderr
|
23
|
+
def initialize(dest, opts = {})
|
24
|
+
@user, @host, @port = split_destination(dest)
|
25
|
+
if !(@host && @user)
|
26
|
+
raise "Invalid destination of ssh server."
|
27
|
+
end
|
28
|
+
@shell = opts[:shell] || 'bash'
|
29
|
+
@rvm = opts[:rvm]
|
30
|
+
@rvm_init = opts[:rvm_init]
|
31
|
+
if (@rvm || @rvm_init) && !(String === @rvm_init)
|
32
|
+
@rvm_init = DEFAULT_RVM_SCRIPT
|
33
|
+
end
|
34
|
+
@out = opts[:output] || DEFAULT_OUTPUT_FILE
|
35
|
+
@directory = opts[:dir]
|
36
|
+
end
|
37
|
+
|
38
|
+
def split_destination(dest)
|
39
|
+
if (n = dest.index("@")) && n > 0
|
40
|
+
user = dest[0..(n - 1)]
|
41
|
+
host_dir = dest[(n + 1)..-1]
|
42
|
+
else
|
43
|
+
raise "Not include '@': #{dest}"
|
44
|
+
end
|
45
|
+
if n = host_dir.index(':')
|
46
|
+
host = host_dir[0..(n - 1)]
|
47
|
+
port = host_dir[(n + 1)..-1]
|
48
|
+
else
|
49
|
+
host = host_dir
|
50
|
+
port = nil
|
51
|
+
end
|
52
|
+
[user && user.size > 0 ? user : nil,
|
53
|
+
host && host.size > 0 ? host : nil,
|
54
|
+
port && port.size > 0 ? port.to_i : nil]
|
55
|
+
end
|
56
|
+
private :split_destination
|
57
|
+
|
58
|
+
def shell_exec(sh, cmd)
|
59
|
+
pr = sh.execute!(cmd)
|
60
|
+
if pr.exit_status != 0
|
61
|
+
raise GetInvalidExitStatus, "Can not execute '#{cmd}' on #{@host} properly."
|
62
|
+
end
|
63
|
+
end
|
64
|
+
private :shell_exec
|
65
|
+
|
66
|
+
def execute_command(*cmds)
|
67
|
+
Net::SSH.start(@host, @user, :port => @port) do |ssh|
|
68
|
+
ssh.shell(@shell) do |sh|
|
69
|
+
shell_exec(sh, "cd #{@directory}") if @directory
|
70
|
+
shell_exec(sh, "source #{@rvm_init}") if @rvm_init
|
71
|
+
shell_exec(sh, "rvm use #{@rvm}") if @rvm
|
72
|
+
cmds.each do |c|
|
73
|
+
sh.execute c
|
74
|
+
end
|
75
|
+
sh.execute "exit"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
private :execute_command
|
80
|
+
|
81
|
+
def get_environment
|
82
|
+
execute_command('echo "directory: " `pwd`',
|
83
|
+
'echo "files:"',
|
84
|
+
'ls',
|
85
|
+
'if which rvm > /dev/null; then rvm info; else ruby -v; fi')
|
86
|
+
end
|
87
|
+
|
88
|
+
def start(*args)
|
89
|
+
execute_command("nohup #{args.join(' ')} > #{@out} 2>&1 &")
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
data/lib/drbqs/task.rb
CHANGED
@@ -8,7 +8,7 @@ module DRbQS
|
|
8
8
|
attr_reader :hook
|
9
9
|
|
10
10
|
# Nodes execute obj.method_sym(*args).
|
11
|
-
# Server executes &hook with a server instance and an
|
11
|
+
# Server executes &hook with a server instance and an object of result
|
12
12
|
# after the server accepts the results from nodes.
|
13
13
|
def initialize(obj, method_sym, args = [], &hook)
|
14
14
|
begin
|
@@ -35,4 +35,29 @@ module DRbQS
|
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
38
|
+
class CommandExecute
|
39
|
+
def initialize(cmd)
|
40
|
+
@cmd = cmd
|
41
|
+
unless (Array === @cmd || String === @cmd)
|
42
|
+
raise ArgumentError, "Invalid command: #{@cmd.inspect}"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def exec
|
47
|
+
case @cmd
|
48
|
+
when Array
|
49
|
+
@cmd.each { |c| system(c) }
|
50
|
+
when String
|
51
|
+
system(@cmd)
|
52
|
+
end
|
53
|
+
$?.exitstatus
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
class CommandTask < Task
|
58
|
+
# &hook takes a server instance and exit number of command.
|
59
|
+
def initialize(cmd, &hook)
|
60
|
+
super(CommandExecute.new(cmd), :exec, &hook)
|
61
|
+
end
|
62
|
+
end
|
38
63
|
end
|
data/lib/drbqs/task_generator.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
module DRbQS
|
2
|
+
class DRbQS::TaskCreatingError < StandardError
|
3
|
+
end
|
4
|
+
|
2
5
|
class TaskGenerator
|
3
6
|
def initialize(data = {})
|
4
7
|
data.each do |key, val|
|
@@ -32,7 +35,11 @@ module DRbQS
|
|
32
35
|
@__iterate__ = iterate
|
33
36
|
@__fiber_init__ = lambda do
|
34
37
|
@__fiber__ = Fiber.new do
|
35
|
-
|
38
|
+
begin
|
39
|
+
instance_eval(&block)
|
40
|
+
rescue => err
|
41
|
+
raise DRbQS::TaskCreatingError, "\n #{err.to_s}"
|
42
|
+
end
|
36
43
|
nil
|
37
44
|
end
|
38
45
|
end
|
@@ -68,8 +75,12 @@ module DRbQS
|
|
68
75
|
nil
|
69
76
|
end
|
70
77
|
|
78
|
+
DEBUG_TASK_PROGRESS = 1000
|
79
|
+
|
71
80
|
# Create all tasks for test and return [group_number, task_number] if all tasks created properly.
|
72
|
-
def debug_all_tasks(
|
81
|
+
def debug_all_tasks(opts = {})
|
82
|
+
limit = opts[:limit]
|
83
|
+
progress = opts[:progress]
|
73
84
|
group_number = 0
|
74
85
|
task_number = 0
|
75
86
|
while ary = new_tasks
|
@@ -78,6 +89,12 @@ module DRbQS
|
|
78
89
|
raise "Invalid #{i}th task: #{t.inspect}"
|
79
90
|
end
|
80
91
|
task_number += 1
|
92
|
+
if progress && (task_number % DEBUG_TASK_PROGRESS == 0)
|
93
|
+
puts "#{task_number} tasks have been created."
|
94
|
+
end
|
95
|
+
if limit && task_number > limit
|
96
|
+
break
|
97
|
+
end
|
81
98
|
end
|
82
99
|
group_number += 1
|
83
100
|
end
|
data/lib/drbqs.rb
CHANGED
@@ -14,6 +14,10 @@ module DRbQS
|
|
14
14
|
autoload :Task, 'drbqs/task'
|
15
15
|
autoload :TaskGenerator, 'drbqs/task_generator'
|
16
16
|
autoload :Manage, 'drbqs/manage'
|
17
|
+
autoload :Config, 'drbqs/config'
|
18
|
+
autoload :SSHShell, 'drbqs/ssh_shell'
|
19
|
+
autoload :CommandTask, 'drbqs/task'
|
20
|
+
autoload :CommandExecute, 'drbqs/task'
|
17
21
|
|
18
22
|
ROOT_DEFAULT_PORT = 13500
|
19
23
|
end
|
data/spec/config_spec.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
describe DRbQS::Config do
|
4
|
+
it "should return nil" do
|
5
|
+
DRbQS::Config.set_directory(File.dirname(__FILE__) + '/not_exist_path')
|
6
|
+
DRbQS::Config.get_acl_file.should be_nil
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should return existing path" do
|
10
|
+
path = File.dirname(__FILE__) + '/data'
|
11
|
+
DRbQS::Config.set_directory(path)
|
12
|
+
DRbQS::Config.get_acl_file.should == File.join(path, 'acl.txt')
|
13
|
+
end
|
14
|
+
end
|
data/spec/manage_spec.rb
CHANGED
@@ -27,12 +27,12 @@ describe DRbQS do
|
|
27
27
|
|
28
28
|
@uri = 'druby://:13501'
|
29
29
|
|
30
|
-
@manage = DRbQS::Manage.new
|
30
|
+
@manage = DRbQS::Manage.new
|
31
31
|
end
|
32
32
|
|
33
33
|
it "should send exit signal" do
|
34
34
|
lambda do
|
35
|
-
@manage.send_exit_signal
|
35
|
+
@manage.send_exit_signal(@uri)
|
36
36
|
end.should_not raise_error
|
37
37
|
lambda do
|
38
38
|
i = 0
|
@@ -47,4 +47,11 @@ describe DRbQS do
|
|
47
47
|
end.should_not raise_error
|
48
48
|
end
|
49
49
|
|
50
|
+
it "should split arguments" do
|
51
|
+
ary = ['abc', 'def', '--', '123', '45', '6']
|
52
|
+
a1, a2 = DRbQS::Manage.split_arguments(ary)
|
53
|
+
a1.should == ['abc', 'def']
|
54
|
+
a2.should == ['123', '45', '6']
|
55
|
+
end
|
56
|
+
|
50
57
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
+
|
3
|
+
require 'drbqs/ssh_shell'
|
4
|
+
|
5
|
+
describe DRbQS::SSHShell do
|
6
|
+
it "should split destination" do
|
7
|
+
ssh = DRbQS::SSHShell.new('user@hostname')
|
8
|
+
ssh.user.should == 'user'
|
9
|
+
ssh.host.should == 'hostname'
|
10
|
+
ssh.port.should be_nil
|
11
|
+
ssh.directory.should be_nil
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should split destination including directory" do
|
15
|
+
ssh = DRbQS::SSHShell.new('user@hostname:22', :dir => '/path/to/directory')
|
16
|
+
ssh.user.should == 'user'
|
17
|
+
ssh.host.should == 'hostname'
|
18
|
+
ssh.port.should == 22
|
19
|
+
ssh.directory.should == '/path/to/directory'
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should raise error: not include '@'" do
|
23
|
+
lambda do
|
24
|
+
DRbQS::SSHShell.new('userhostname')
|
25
|
+
end.should raise_error
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should raise error: empty user name" do
|
29
|
+
lambda do
|
30
|
+
DRbQS::SSHShell.new('@hostname')
|
31
|
+
end.should raise_error
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should raise error: empty host name" do
|
35
|
+
lambda do
|
36
|
+
DRbQS::SSHShell.new('user:22')
|
37
|
+
end.should raise_error
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
metadata
CHANGED
@@ -1,83 +1,85 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: drbqs
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.7
|
3
|
+
version: !ruby/object:Gem::Version
|
5
4
|
prerelease:
|
5
|
+
version: 0.0.8
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Takayuki YAMAGUCHI
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
12
|
+
|
13
|
+
date: 2011-03-20 00:00:00 +09:00
|
13
14
|
default_executable:
|
14
|
-
dependencies:
|
15
|
-
- !ruby/object:Gem::Dependency
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
16
17
|
name: rspec
|
17
|
-
requirement: &
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
18
19
|
none: false
|
19
|
-
requirements:
|
20
|
-
- -
|
21
|
-
- !ruby/object:Gem::Version
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
22
23
|
version: 2.5.0
|
23
24
|
type: :development
|
24
25
|
prerelease: false
|
25
|
-
version_requirements: *
|
26
|
-
- !ruby/object:Gem::Dependency
|
26
|
+
version_requirements: *id001
|
27
|
+
- !ruby/object:Gem::Dependency
|
27
28
|
name: yard
|
28
|
-
requirement: &
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
29
30
|
none: false
|
30
|
-
requirements:
|
31
|
+
requirements:
|
31
32
|
- - ~>
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
+
- !ruby/object:Gem::Version
|
33
34
|
version: 0.6.0
|
34
35
|
type: :development
|
35
36
|
prerelease: false
|
36
|
-
version_requirements: *
|
37
|
-
- !ruby/object:Gem::Dependency
|
37
|
+
version_requirements: *id002
|
38
|
+
- !ruby/object:Gem::Dependency
|
38
39
|
name: bundler
|
39
|
-
requirement: &
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
40
41
|
none: false
|
41
|
-
requirements:
|
42
|
+
requirements:
|
42
43
|
- - ~>
|
43
|
-
- !ruby/object:Gem::Version
|
44
|
+
- !ruby/object:Gem::Version
|
44
45
|
version: 1.0.0
|
45
46
|
type: :development
|
46
47
|
prerelease: false
|
47
|
-
version_requirements: *
|
48
|
-
- !ruby/object:Gem::Dependency
|
48
|
+
version_requirements: *id003
|
49
|
+
- !ruby/object:Gem::Dependency
|
49
50
|
name: jeweler
|
50
|
-
requirement: &
|
51
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
51
52
|
none: false
|
52
|
-
requirements:
|
53
|
+
requirements:
|
53
54
|
- - ~>
|
54
|
-
- !ruby/object:Gem::Version
|
55
|
+
- !ruby/object:Gem::Version
|
55
56
|
version: 1.5.2
|
56
57
|
type: :development
|
57
58
|
prerelease: false
|
58
|
-
version_requirements: *
|
59
|
-
- !ruby/object:Gem::Dependency
|
59
|
+
version_requirements: *id004
|
60
|
+
- !ruby/object:Gem::Dependency
|
60
61
|
name: rcov
|
61
|
-
requirement: &
|
62
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
62
63
|
none: false
|
63
|
-
requirements:
|
64
|
-
- -
|
65
|
-
- !ruby/object:Gem::Version
|
66
|
-
version:
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: "0"
|
67
68
|
type: :development
|
68
69
|
prerelease: false
|
69
|
-
version_requirements: *
|
70
|
+
version_requirements: *id005
|
70
71
|
description: Task queuing system over network that is implemented by dRuby.
|
71
72
|
email: d@ytak.info
|
72
|
-
executables:
|
73
|
+
executables:
|
73
74
|
- drbqs-manage
|
74
75
|
- drbqs-node
|
75
76
|
- drbqs-server
|
76
77
|
extensions: []
|
77
|
-
|
78
|
+
|
79
|
+
extra_rdoc_files:
|
78
80
|
- LICENSE.txt
|
79
81
|
- README.rdoc
|
80
|
-
files:
|
82
|
+
files:
|
81
83
|
- .document
|
82
84
|
- .rspec
|
83
85
|
- Gemfile
|
@@ -90,15 +92,19 @@ files:
|
|
90
92
|
- bin/drbqs-server
|
91
93
|
- drbqs.gemspec
|
92
94
|
- example/README.md
|
95
|
+
- example/command/server_def.rb
|
93
96
|
- example/drbqs-manage-test.rb
|
94
97
|
- example/drbqs-node-test.rb
|
95
98
|
- example/drbqs-server-test.rb
|
96
|
-
- example/server/
|
99
|
+
- example/server/server.rb
|
97
100
|
- example/sum/server_def.rb
|
98
101
|
- example/sum/sum.rb
|
102
|
+
- example/sum2/server_def.rb
|
103
|
+
- example/sum2/sum.rb
|
99
104
|
- lib/drbqs.rb
|
100
105
|
- lib/drbqs/acl_file.rb
|
101
106
|
- lib/drbqs/client.rb
|
107
|
+
- lib/drbqs/config.rb
|
102
108
|
- lib/drbqs/connection.rb
|
103
109
|
- lib/drbqs/manage.rb
|
104
110
|
- lib/drbqs/message.rb
|
@@ -107,10 +113,12 @@ files:
|
|
107
113
|
- lib/drbqs/server.rb
|
108
114
|
- lib/drbqs/server_define.rb
|
109
115
|
- lib/drbqs/server_hook.rb
|
116
|
+
- lib/drbqs/ssh_shell.rb
|
110
117
|
- lib/drbqs/task.rb
|
111
118
|
- lib/drbqs/task_client.rb
|
112
119
|
- lib/drbqs/task_generator.rb
|
113
120
|
- spec/acl_file_spec.rb
|
121
|
+
- spec/config_spec.rb
|
114
122
|
- spec/connection_spec.rb
|
115
123
|
- spec/data/acl.txt
|
116
124
|
- spec/manage_spec.rb
|
@@ -121,6 +129,7 @@ files:
|
|
121
129
|
- spec/server_hook_spec.rb
|
122
130
|
- spec/server_spec.rb
|
123
131
|
- spec/spec_helper.rb
|
132
|
+
- spec/ssh_shell_spec.rb
|
124
133
|
- spec/task_client_spec.rb
|
125
134
|
- spec/task_generator_spec.rb
|
126
135
|
- spec/task_spec.rb
|
@@ -129,35 +138,38 @@ files:
|
|
129
138
|
- spec/test2_spec.rb
|
130
139
|
has_rdoc: true
|
131
140
|
homepage: http://github.com/ytaka/drbqs
|
132
|
-
licenses:
|
141
|
+
licenses:
|
133
142
|
- GPL3
|
134
143
|
post_install_message:
|
135
144
|
rdoc_options: []
|
136
|
-
|
145
|
+
|
146
|
+
require_paths:
|
137
147
|
- lib
|
138
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
148
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
139
149
|
none: false
|
140
|
-
requirements:
|
141
|
-
- -
|
142
|
-
- !ruby/object:Gem::Version
|
143
|
-
|
144
|
-
segments:
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
hash: 3292588984449245047
|
154
|
+
segments:
|
145
155
|
- 0
|
146
|
-
|
147
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
156
|
+
version: "0"
|
157
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
148
158
|
none: false
|
149
|
-
requirements:
|
150
|
-
- -
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version:
|
159
|
+
requirements:
|
160
|
+
- - ">="
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: "0"
|
153
163
|
requirements: []
|
164
|
+
|
154
165
|
rubyforge_project:
|
155
|
-
rubygems_version: 1.
|
166
|
+
rubygems_version: 1.5.2
|
156
167
|
signing_key:
|
157
168
|
specification_version: 3
|
158
169
|
summary: dRuby Queueing System
|
159
|
-
test_files:
|
170
|
+
test_files:
|
160
171
|
- spec/acl_file_spec.rb
|
172
|
+
- spec/config_spec.rb
|
161
173
|
- spec/connection_spec.rb
|
162
174
|
- spec/manage_spec.rb
|
163
175
|
- spec/message_spec.rb
|
@@ -167,6 +179,7 @@ test_files:
|
|
167
179
|
- spec/server_hook_spec.rb
|
168
180
|
- spec/server_spec.rb
|
169
181
|
- spec/spec_helper.rb
|
182
|
+
- spec/ssh_shell_spec.rb
|
170
183
|
- spec/task_client_spec.rb
|
171
184
|
- spec/task_generator_spec.rb
|
172
185
|
- spec/task_spec.rb
|
File without changes
|