drbqs 0.0.8 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +151 -0
- data/Rakefile +3 -2
- data/VERSION +1 -1
- data/bin/drbqs-manage +27 -7
- data/bin/drbqs-node +1 -0
- data/drbqs.gemspec +25 -7
- data/example/transfer/file.rb +46 -0
- data/example/transfer/server_def.rb +19 -0
- data/lib/drbqs.rb +5 -2
- data/lib/drbqs/client.rb +14 -1
- data/lib/drbqs/config.rb +15 -3
- data/lib/drbqs/manage.rb +50 -3
- data/lib/drbqs/message.rb +26 -1
- data/lib/drbqs/node_list.rb +25 -0
- data/lib/drbqs/queue.rb +1 -0
- data/lib/drbqs/server.rb +10 -2
- data/lib/drbqs/ssh/host.rb +26 -0
- data/lib/drbqs/{ssh_shell.rb → ssh/shell.rb} +64 -17
- data/lib/drbqs/ssh/transfer.rb +70 -0
- data/lib/drbqs/task.rb +17 -4
- data/lib/drbqs/utils/filename.rb +118 -0
- data/spec/filename_spec.rb +44 -0
- data/spec/message_spec.rb +10 -0
- data/spec/node_list_spec.rb +40 -0
- data/spec/server_define_spec.rb +23 -12
- data/spec/ssh_shell_spec.rb +1 -1
- data/spec/task_spec.rb +13 -0
- data/spec/transfer_spec.rb +13 -0
- metadata +48 -6
- data/README.rdoc +0 -128
data/README.md
ADDED
@@ -0,0 +1,151 @@
|
|
1
|
+
# drbqs
|
2
|
+
|
3
|
+
Task queuing system over network that is implemented by dRuby.
|
4
|
+
|
5
|
+
## Summary
|
6
|
+
|
7
|
+
To use DRbQS, first, we start a server of DRbQS.
|
8
|
+
Second, we execute nodes (on same host or other hosts)
|
9
|
+
loading libraries required by the server
|
10
|
+
and connect the nodes to the server.
|
11
|
+
|
12
|
+
The behavior of nodes requests tasks, gets tasks from a server, processes the tasks,
|
13
|
+
and sends results of the tasks to the server.
|
14
|
+
The nodes work repeatedly until they get exit signal from server.
|
15
|
+
The server prepares tasks and checks that nodes are alive.
|
16
|
+
If the server does not communicate with the nodes unexpectedly,
|
17
|
+
the server deletes the nodes from node list and
|
18
|
+
requeues their calculating tasks.
|
19
|
+
We can set hooks for tasks.
|
20
|
+
The hooks are executed for results of the tasks
|
21
|
+
after the server accepts them from nodes.
|
22
|
+
|
23
|
+
The tasks are made from objects, an instance method of them, and its arguments.
|
24
|
+
Because we use Marshal.dump and Marshal.load for communication of a server and nodes,
|
25
|
+
the objects and the arguments must be marshalized.
|
26
|
+
And also we tell the server and the nodes the definision of class of the objects and the arguments.
|
27
|
+
|
28
|
+
## Requirements
|
29
|
+
|
30
|
+
DRbQS uses Fiber, so ruby requires version 1.9.
|
31
|
+
And we use net-ssh and net-ssh-shell to execute servers and nodes over ssh.
|
32
|
+
|
33
|
+
## Usage
|
34
|
+
|
35
|
+
### Preparation
|
36
|
+
|
37
|
+
We prepare a class to send tasks over network,
|
38
|
+
which has data and a method to deal with tasks.
|
39
|
+
|
40
|
+
We make sum.rb as the following.
|
41
|
+
|
42
|
+
class Sum
|
43
|
+
def initialize(start_num, end_num)
|
44
|
+
@num = [start_num, end_num]
|
45
|
+
end
|
46
|
+
|
47
|
+
def exec
|
48
|
+
(@num[0]..@num[1]).inject(0) { |sum, i| sum += i }
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
The Sum class calculates sum of numbers from start_num to end_num.
|
53
|
+
The task we want to calculate is summation of numbers.
|
54
|
+
|
55
|
+
### Start server
|
56
|
+
|
57
|
+
We make server.rb as the following.
|
58
|
+
|
59
|
+
require_relative 'sum.rb'
|
60
|
+
|
61
|
+
DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
|
62
|
+
10.step(100, 10) do |i|
|
63
|
+
task = DRbQS::Task.new(Sum.new(i - 10, i), :exec)
|
64
|
+
server.queue.add(task)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
In terminal, we load server.rb and execute server of drbqs.
|
69
|
+
|
70
|
+
drbqs-server server.rb -p 13500
|
71
|
+
|
72
|
+
### Hook of server
|
73
|
+
|
74
|
+
We can use two hooks of server: 'empty_queue' and 'finish'.
|
75
|
+
|
76
|
+
DRbQS.define_server do |server, argv, opts|
|
77
|
+
server.add_hook(:empty_queue) do |srv|
|
78
|
+
srv.queue.add( ... )
|
79
|
+
end
|
80
|
+
|
81
|
+
server.add_hook(:finish) do |srv|
|
82
|
+
srv.exit
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
'finish' hook usually exit server program, but
|
87
|
+
an option :finish_exit for DRbQS.define_server or DRbQS.new
|
88
|
+
is nearly same.
|
89
|
+
|
90
|
+
We can use 'empty_queue' hook for adding tasks
|
91
|
+
when task queue is empty.
|
92
|
+
|
93
|
+
### Task generator
|
94
|
+
|
95
|
+
Arguments of DRbQS::TaskGenerator.new define instance variables.
|
96
|
+
|
97
|
+
task_generator = DRbQS::TaskGenerator.new(:abc => 'ABC', :def => 123, :data => [1, 2, 3])
|
98
|
+
|
99
|
+
The above example defines the following instance variables.
|
100
|
+
|
101
|
+
@abc = 'ABC'
|
102
|
+
@def = 123
|
103
|
+
@data = [1, 2, 3]
|
104
|
+
|
105
|
+
Then, DRbQS::TaskGenerator#set method defines generation of tasks.
|
106
|
+
The block of the method is evaluated in the context of task_generator.
|
107
|
+
For the above example we can use @abc, @def, and @data.
|
108
|
+
|
109
|
+
task_generator.set do
|
110
|
+
@data.each do |i|
|
111
|
+
create_add_task(i, :to_s)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
DRbQS::TaskGenerator#create_add_task creates a task
|
116
|
+
and the task is returned by DRbQS::TaskGenerator#new_tasks.
|
117
|
+
The arguments of DRbQS::TaskGenerator#create_add_task is
|
118
|
+
the same as DRbQS::Task.new.
|
119
|
+
|
120
|
+
To use the generator in DRbQS::Server,
|
121
|
+
we set the generator by DRbQS::Server#add_task_generator.
|
122
|
+
|
123
|
+
### Start node and connect server
|
124
|
+
|
125
|
+
Because nodes needs class Sum,
|
126
|
+
the nodes load sum.rb when they starts.
|
127
|
+
Then, we type in terminal.
|
128
|
+
|
129
|
+
drbqs-node druby://localhost:13500/ -l sum.rb
|
130
|
+
|
131
|
+
To use two cpu cores we execute two processes by the following.
|
132
|
+
|
133
|
+
drbqs-node 2 druby://localhost:13500/ -l sum.rb
|
134
|
+
|
135
|
+
Then, if it succeeds, the calculation starts.
|
136
|
+
If it finishes, the server and node ends.
|
137
|
+
|
138
|
+
## Contributing to drbqs
|
139
|
+
|
140
|
+
- Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
|
141
|
+
- Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
|
142
|
+
- Fork the project
|
143
|
+
- Start a feature/bugfix branch
|
144
|
+
- Commit and push until you are happy with your contribution
|
145
|
+
- Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
|
146
|
+
- Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
|
147
|
+
|
148
|
+
## Copyright
|
149
|
+
|
150
|
+
Copyright (c) 2011 Takayuki YAMAGUCHI. See LICENSE.txt for
|
151
|
+
further details.
|
data/Rakefile
CHANGED
@@ -21,8 +21,9 @@ Jeweler::Tasks.new do |gem|
|
|
21
21
|
gem.authors = ["Takayuki YAMAGUCHI"]
|
22
22
|
# Include your dependencies below. Runtime dependencies are required when using your gem,
|
23
23
|
# and development dependencies are only needed for development (ie running rake tasks, tests, etc)
|
24
|
-
|
25
|
-
|
24
|
+
gem.add_runtime_dependency 'net-ssh', '>= 2.1.3'
|
25
|
+
gem.add_runtime_dependency 'net-ssh-shell', '>= 0.1.0'
|
26
|
+
gem.add_development_dependency 'rspec', '>= 2.5.0'
|
26
27
|
end
|
27
28
|
Jeweler::RubygemsDotOrgTasks.new
|
28
29
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.9
|
data/bin/drbqs-manage
CHANGED
@@ -12,10 +12,23 @@ Manage DRbQS server by sending messages.
|
|
12
12
|
|
13
13
|
HELP
|
14
14
|
|
15
|
+
def check_argument_size(argv, check_method, n)
|
16
|
+
unless argv.size.__send__(check_method, n)
|
17
|
+
raise "Invalid arguments number. Please refer '#{File.basename(__FILE__)} -h'."
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
15
21
|
command = ARGV[0]
|
16
22
|
argv, command_args = DRbQS::Manage.split_arguments(ARGV)
|
17
23
|
|
18
24
|
options = {}
|
25
|
+
if /^ssh/ =~ command
|
26
|
+
check_argument_size(argv, :>=, 2)
|
27
|
+
ssh_host =DRbQS::SSHHost.new
|
28
|
+
path, options = ssh_host.get_options(argv[1])
|
29
|
+
$stdout.puts "Use configuration: #{path}" if path
|
30
|
+
argv[1] = options[:dest] if options[:dest]
|
31
|
+
end
|
19
32
|
|
20
33
|
begin
|
21
34
|
OptionParser.new(help_message) do |opt|
|
@@ -37,6 +50,12 @@ begin
|
|
37
50
|
opt.on('--output PATH', String, 'File path that stdout and stderr are output to over ssh.') do |v|
|
38
51
|
options[:output] = v
|
39
52
|
end
|
53
|
+
opt.on('--nice NUM', Integer, 'Set the value for nice command.') do |v|
|
54
|
+
options[:nice] = v
|
55
|
+
end
|
56
|
+
opt.on('--nohup', 'Use nohup command.') do |v|
|
57
|
+
options[:nohup] = true
|
58
|
+
end
|
40
59
|
opt.parse!(argv)
|
41
60
|
end
|
42
61
|
rescue OptionParser::InvalidOption
|
@@ -53,12 +72,6 @@ MES
|
|
53
72
|
exit(2)
|
54
73
|
end
|
55
74
|
|
56
|
-
def check_argument_size(argv, check_method, n)
|
57
|
-
unless argv.size.__send__(check_method, n)
|
58
|
-
raise "Invalid arguments number. Please refer '#{File.basename(__FILE__)} -h'."
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
75
|
manage = DRbQS::Manage.new
|
63
76
|
|
64
77
|
case command
|
@@ -66,13 +79,20 @@ when 'exit-signal'
|
|
66
79
|
check_argument_size(argv, :==, 2)
|
67
80
|
uri = argv[1]
|
68
81
|
manage.send_exit_signal(uri)
|
82
|
+
when 'status'
|
83
|
+
check_argument_size(argv, :==, 2)
|
84
|
+
uri = argv[1]
|
85
|
+
$stdout.puts manage.get_status(uri)
|
69
86
|
when 'initialize'
|
70
87
|
check_argument_size(argv, :==, 1)
|
71
88
|
manage.create_config
|
72
89
|
when 'ssh'
|
73
|
-
check_argument_size(argv, :>=, 2)
|
74
90
|
dest = argv[1]
|
75
91
|
manage.execute_over_ssh(dest, options, command_args)
|
92
|
+
when 'new-filename'
|
93
|
+
check_argument_size(argv, :==, 2)
|
94
|
+
fname = DRbQS::FileName.new(argv[1], :add => :auto, :type => :time, :position => :suffix)
|
95
|
+
$stdout.puts fname.create(:directory => true)
|
76
96
|
else
|
77
97
|
raise "Invalid command: #{command}"
|
78
98
|
end
|
data/bin/drbqs-node
CHANGED
data/drbqs.gemspec
CHANGED
@@ -5,24 +5,24 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{drbqs}
|
8
|
-
s.version = "0.0.
|
8
|
+
s.version = "0.0.9"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Takayuki YAMAGUCHI"]
|
12
|
-
s.date = %q{2011-03-
|
12
|
+
s.date = %q{2011-03-22}
|
13
13
|
s.description = %q{Task queuing system over network that is implemented by dRuby.}
|
14
14
|
s.email = %q{d@ytak.info}
|
15
15
|
s.executables = ["drbqs-manage", "drbqs-node", "drbqs-server"]
|
16
16
|
s.extra_rdoc_files = [
|
17
17
|
"LICENSE.txt",
|
18
|
-
"README.
|
18
|
+
"README.md"
|
19
19
|
]
|
20
20
|
s.files = [
|
21
21
|
".document",
|
22
22
|
".rspec",
|
23
23
|
"Gemfile",
|
24
24
|
"LICENSE.txt",
|
25
|
-
"README.
|
25
|
+
"README.md",
|
26
26
|
"Rakefile",
|
27
27
|
"VERSION",
|
28
28
|
"bin/drbqs-manage",
|
@@ -39,6 +39,8 @@ Gem::Specification.new do |s|
|
|
39
39
|
"example/sum/sum.rb",
|
40
40
|
"example/sum2/server_def.rb",
|
41
41
|
"example/sum2/sum.rb",
|
42
|
+
"example/transfer/file.rb",
|
43
|
+
"example/transfer/server_def.rb",
|
42
44
|
"lib/drbqs.rb",
|
43
45
|
"lib/drbqs/acl_file.rb",
|
44
46
|
"lib/drbqs/client.rb",
|
@@ -51,14 +53,18 @@ Gem::Specification.new do |s|
|
|
51
53
|
"lib/drbqs/server.rb",
|
52
54
|
"lib/drbqs/server_define.rb",
|
53
55
|
"lib/drbqs/server_hook.rb",
|
54
|
-
"lib/drbqs/
|
56
|
+
"lib/drbqs/ssh/host.rb",
|
57
|
+
"lib/drbqs/ssh/shell.rb",
|
58
|
+
"lib/drbqs/ssh/transfer.rb",
|
55
59
|
"lib/drbqs/task.rb",
|
56
60
|
"lib/drbqs/task_client.rb",
|
57
61
|
"lib/drbqs/task_generator.rb",
|
62
|
+
"lib/drbqs/utils/filename.rb",
|
58
63
|
"spec/acl_file_spec.rb",
|
59
64
|
"spec/config_spec.rb",
|
60
65
|
"spec/connection_spec.rb",
|
61
66
|
"spec/data/acl.txt",
|
67
|
+
"spec/filename_spec.rb",
|
62
68
|
"spec/manage_spec.rb",
|
63
69
|
"spec/message_spec.rb",
|
64
70
|
"spec/node_list_spec.rb",
|
@@ -73,7 +79,8 @@ Gem::Specification.new do |s|
|
|
73
79
|
"spec/task_spec.rb",
|
74
80
|
"spec/test/test1.rb",
|
75
81
|
"spec/test1_spec.rb",
|
76
|
-
"spec/test2_spec.rb"
|
82
|
+
"spec/test2_spec.rb",
|
83
|
+
"spec/transfer_spec.rb"
|
77
84
|
]
|
78
85
|
s.homepage = %q{http://github.com/ytaka/drbqs}
|
79
86
|
s.licenses = ["GPL3"]
|
@@ -84,6 +91,7 @@ Gem::Specification.new do |s|
|
|
84
91
|
"spec/acl_file_spec.rb",
|
85
92
|
"spec/config_spec.rb",
|
86
93
|
"spec/connection_spec.rb",
|
94
|
+
"spec/filename_spec.rb",
|
87
95
|
"spec/manage_spec.rb",
|
88
96
|
"spec/message_spec.rb",
|
89
97
|
"spec/node_list_spec.rb",
|
@@ -98,7 +106,8 @@ Gem::Specification.new do |s|
|
|
98
106
|
"spec/task_spec.rb",
|
99
107
|
"spec/test/test1.rb",
|
100
108
|
"spec/test1_spec.rb",
|
101
|
-
"spec/test2_spec.rb"
|
109
|
+
"spec/test2_spec.rb",
|
110
|
+
"spec/transfer_spec.rb"
|
102
111
|
]
|
103
112
|
|
104
113
|
if s.respond_to? :specification_version then
|
@@ -110,12 +119,18 @@ Gem::Specification.new do |s|
|
|
110
119
|
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
111
120
|
s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
|
112
121
|
s.add_development_dependency(%q<rcov>, [">= 0"])
|
122
|
+
s.add_runtime_dependency(%q<net-ssh>, [">= 2.1.3"])
|
123
|
+
s.add_runtime_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
|
124
|
+
s.add_development_dependency(%q<rspec>, [">= 2.5.0"])
|
113
125
|
else
|
114
126
|
s.add_dependency(%q<rspec>, [">= 2.5.0"])
|
115
127
|
s.add_dependency(%q<yard>, ["~> 0.6.0"])
|
116
128
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
117
129
|
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
118
130
|
s.add_dependency(%q<rcov>, [">= 0"])
|
131
|
+
s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
|
132
|
+
s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
|
133
|
+
s.add_dependency(%q<rspec>, [">= 2.5.0"])
|
119
134
|
end
|
120
135
|
else
|
121
136
|
s.add_dependency(%q<rspec>, [">= 2.5.0"])
|
@@ -123,6 +138,9 @@ Gem::Specification.new do |s|
|
|
123
138
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
124
139
|
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
125
140
|
s.add_dependency(%q<rcov>, [">= 0"])
|
141
|
+
s.add_dependency(%q<net-ssh>, [">= 2.1.3"])
|
142
|
+
s.add_dependency(%q<net-ssh-shell>, [">= 0.1.0"])
|
143
|
+
s.add_dependency(%q<rspec>, [">= 2.5.0"])
|
126
144
|
end
|
127
145
|
end
|
128
146
|
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
class CreateFile
|
4
|
+
def initialize(n)
|
5
|
+
@n = n
|
6
|
+
end
|
7
|
+
|
8
|
+
def output_to_file
|
9
|
+
path = "/tmp/transfer_test_#{@n.to_s}.txt"
|
10
|
+
open(path, 'w') { |f| f.puts "file#{@n.to_s}" }
|
11
|
+
path
|
12
|
+
end
|
13
|
+
|
14
|
+
def create
|
15
|
+
DRbQS::FileTransfer.enqueue(output_to_file)
|
16
|
+
nil
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_compress
|
20
|
+
DRbQS::FileTransfer.compress_enqueue(output_to_file)
|
21
|
+
nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class CreateDirectory
|
26
|
+
def initialize(n)
|
27
|
+
@n = n
|
28
|
+
end
|
29
|
+
|
30
|
+
def output_to_directory
|
31
|
+
path = "/tmp/transfer_test_#{@n.to_s}/"
|
32
|
+
FileUtils.mkdir_p(path)
|
33
|
+
open(File.join(path, 'tmp.txt'), 'w') { |f| f.puts "file#{@n.to_s}" }
|
34
|
+
path
|
35
|
+
end
|
36
|
+
|
37
|
+
def create
|
38
|
+
DRbQS::FileTransfer.enqueue(output_to_directory)
|
39
|
+
nil
|
40
|
+
end
|
41
|
+
|
42
|
+
def create_compress
|
43
|
+
DRbQS::FileTransfer.compress_enqueue(output_to_directory)
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
#
|
2
|
+
# Usage:
|
3
|
+
# drbqs-server server_def.rb
|
4
|
+
#
|
5
|
+
|
6
|
+
require_relative 'file.rb'
|
7
|
+
|
8
|
+
DRbQS.define_server(:finish_exit => true) do |server, argv, opts|
|
9
|
+
tgen = DRbQS::TaskGenerator.new(:sleep_time => 2)
|
10
|
+
tgen.set do
|
11
|
+
create_add_task(CreateFile.new(1), :create)
|
12
|
+
create_add_task(CreateFile.new(2), :create_compress)
|
13
|
+
create_add_task(CreateDirectory.new(3), :create)
|
14
|
+
create_add_task(CreateDirectory.new(4), :create_compress)
|
15
|
+
end
|
16
|
+
server.add_task_generator(tgen)
|
17
|
+
|
18
|
+
server.set_file_transfer(ENV['USER'], 'localhost', '/tmp/drbqs_transfer_test/')
|
19
|
+
end
|
data/lib/drbqs.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
require 'thread'
|
2
2
|
require 'logger'
|
3
|
-
require 'fileutils'
|
4
3
|
require 'drb'
|
5
4
|
require 'drb/acl'
|
6
5
|
require 'rinda/tuplespace'
|
@@ -15,9 +14,13 @@ module DRbQS
|
|
15
14
|
autoload :TaskGenerator, 'drbqs/task_generator'
|
16
15
|
autoload :Manage, 'drbqs/manage'
|
17
16
|
autoload :Config, 'drbqs/config'
|
18
|
-
autoload :SSHShell, 'drbqs/
|
17
|
+
autoload :SSHShell, 'drbqs/ssh/shell'
|
18
|
+
autoload :SSHHost, 'drbqs/ssh/host'
|
19
19
|
autoload :CommandTask, 'drbqs/task'
|
20
20
|
autoload :CommandExecute, 'drbqs/task'
|
21
|
+
autoload :FileName, 'drbqs/utils/filename'
|
22
|
+
autoload :Transfer, 'drbqs/ssh/transfer'
|
23
|
+
autoload :FileTransfer, 'drbqs/ssh/transfer'
|
21
24
|
|
22
25
|
ROOT_DEFAULT_PORT = 13500
|
23
26
|
end
|