ruby-cute 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.yardopts +2 -0
- data/Gemfile +6 -0
- data/README.md +137 -6
- data/Rakefile +48 -0
- data/bin/cute +22 -0
- data/debian/changelog +5 -0
- data/debian/compat +1 -0
- data/debian/control +15 -0
- data/debian/copyright +33 -0
- data/debian/ruby-cute.docs +2 -0
- data/debian/ruby-tests.rb +2 -0
- data/debian/rules +19 -0
- data/debian/source/format +1 -0
- data/debian/watch +2 -0
- data/examples/distem-bootstrap +516 -0
- data/examples/g5k_exp1.rb +41 -0
- data/examples/g5k_exp_virt.rb +129 -0
- data/lib/cute.rb +7 -2
- data/lib/cute/bash.rb +337 -0
- data/lib/cute/configparser.rb +404 -0
- data/lib/cute/execute.rb +272 -0
- data/lib/cute/extensions.rb +38 -0
- data/lib/cute/g5k_api.rb +1190 -0
- data/lib/cute/net-ssh.rb +144 -0
- data/lib/cute/net.rb +29 -0
- data/lib/cute/synchronization.rb +89 -0
- data/lib/cute/taktuk.rb +554 -0
- data/lib/cute/version.rb +3 -0
- data/ruby-cute.gemspec +32 -0
- data/spec/extensions_spec.rb +17 -0
- data/spec/g5k_api_spec.rb +192 -0
- data/spec/spec_helper.rb +66 -0
- data/spec/taktuk_spec.rb +129 -0
- data/test/test_bash.rb +71 -0
- metadata +204 -47
data/lib/cute/net-ssh.rb
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'net/ssh/multi'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Net; module SSH
|
5
|
+
|
6
|
+
# The Net::SSH::Multi aims at executing commands in parallel over a set of machines using the SSH protocol.
|
7
|
+
# One of the advantage of this module over {Cute::TakTuk::TakTuk TakTuk} is that it allows to create groups, for example:
|
8
|
+
#
|
9
|
+
# Net::SSH::Multi.start do |session|
|
10
|
+
#
|
11
|
+
# session.group :coord do
|
12
|
+
# session.use("root@#{coordinator}")
|
13
|
+
# end
|
14
|
+
#
|
15
|
+
# session.group :nodes do
|
16
|
+
# nodelist.each{ |node| session.use("root@#{node}")}
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
# # test connection
|
20
|
+
# session.with(:coord).exec! "hostname"
|
21
|
+
# session.with(:nodes).exec! "hostname"
|
22
|
+
#
|
23
|
+
# # Check nfs paths
|
24
|
+
# tmp = session.exec! "ls -a #{ENV['HOME']}"
|
25
|
+
#
|
26
|
+
# # generating ssh password less connection
|
27
|
+
# session.exec! "cat .ssh/id_rsa.pub >> .ssh/authorized_keys"
|
28
|
+
# end
|
29
|
+
#
|
30
|
+
# However, with large set of nodes SSH it is limited an inefficient,
|
31
|
+
# for those cases the best option will be {Cute::TakTuk::TakTuk TakTuk}.
|
32
|
+
# For complete documentation please take a look at
|
33
|
+
# {http://net-ssh.github.io/net-ssh-multi/ Net::SSH::Multi}.
|
34
|
+
# One of the disadvantages of {http://net-ssh.github.io/net-ssh-multi/ Net::SSH::Multi} is that
|
35
|
+
# it does not allow to capture the output (stdout, stderr and status) of executed commands.
|
36
|
+
# Ruby-Cute ships a monkey patch that extends the aforementioned module by adding the method
|
37
|
+
# {Net::SSH::Multi::SessionActions#exec! exec!}
|
38
|
+
# which blocks until the command finishes and captures the output (stdout, stderr and status).
|
39
|
+
#
|
40
|
+
# require 'cute/net-ssh'
|
41
|
+
#
|
42
|
+
# results = {}
|
43
|
+
# Net::SSH::Multi.start do |session|
|
44
|
+
#
|
45
|
+
# # define the servers we want to use
|
46
|
+
# session.use 'user1@host1'
|
47
|
+
# session.use 'user2@host2'
|
48
|
+
#
|
49
|
+
# session.exec "uptime"
|
50
|
+
# session.exec "df"
|
51
|
+
# # execute command, blocks and capture the output
|
52
|
+
# results = session.exec! "date"
|
53
|
+
# # execute commands on a subset of servers
|
54
|
+
# session.exec "hostname"
|
55
|
+
# end
|
56
|
+
# puts results #=> {"node3"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0},
|
57
|
+
# # "node1"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0}, ...}
|
58
|
+
#
|
59
|
+
module Multi
|
60
|
+
|
61
|
+
# sets logger to be used by net-ssh-multi module
|
62
|
+
def self.logger= v
|
63
|
+
@logger = v
|
64
|
+
end
|
65
|
+
|
66
|
+
# @return logger
|
67
|
+
def self.logger
|
68
|
+
if @logger.nil?
|
69
|
+
@logger = Logger.new(STDOUT)
|
70
|
+
logger.level = Logger::INFO
|
71
|
+
end
|
72
|
+
@logger
|
73
|
+
end
|
74
|
+
|
75
|
+
module SessionActions
|
76
|
+
|
77
|
+
# Monkey patch that adds the exec! method.
|
78
|
+
# It executes a command on multiple hosts capturing their associated output (stdout, stderr and status).
|
79
|
+
# It blocks until the command finishes returning the resulting output as a Hash.
|
80
|
+
# It uses a logger for debugging purposes.
|
81
|
+
# @see http://net-ssh.github.io/net-ssh-multi/classes/Net/SSH/Multi/SessionActions.html More information about exec method.
|
82
|
+
# @return [Hash] result Hash stdout, stderr and status of executed commands
|
83
|
+
#
|
84
|
+
# = Example
|
85
|
+
#
|
86
|
+
# session.exec!("date") #=> {"node3"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0},
|
87
|
+
# # "node1"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0}, ...}
|
88
|
+
#
|
89
|
+
# session.exec!("cmd") #=> {"node4"=>{:stderr=>"bash: cmd: command not found", :status=>127},
|
90
|
+
# # "node3"=>{:stderr=>"bash: cmd: command not found", :status=>127}, ...}
|
91
|
+
#
|
92
|
+
def exec!(command, &block)
|
93
|
+
|
94
|
+
results = {}
|
95
|
+
|
96
|
+
main =open_channel do |channel|
|
97
|
+
channel.exec(command) do |ch, success|
|
98
|
+
raise "could not execute command: #{command.inspect} (#{ch[:host]})" unless success
|
99
|
+
Multi.logger.debug("Executing #{command} on [#{ch.connection.host}]")
|
100
|
+
|
101
|
+
results[ch.connection.host] ||= {}
|
102
|
+
|
103
|
+
channel.on_data do |ch, data|
|
104
|
+
if block
|
105
|
+
block.call(ch, :stdout, data)
|
106
|
+
else
|
107
|
+
results[ch.connection.host][:stdout] = data.strip
|
108
|
+
Multi.logger.debug("[#{ch.connection.host}] #{data.strip}")
|
109
|
+
end
|
110
|
+
end
|
111
|
+
channel.on_extended_data do |ch, type, data|
|
112
|
+
if block
|
113
|
+
block.call(ch, :stderr, data)
|
114
|
+
else
|
115
|
+
results[ch.connection.host][:stderr] = data.strip
|
116
|
+
Multi.logger.debug("[#{ch.connection.host}] #{data.strip}")
|
117
|
+
end
|
118
|
+
end
|
119
|
+
channel.on_request("exit-status") do |ch, data|
|
120
|
+
ch[:exit_status] = data.read_long
|
121
|
+
results[ch.connection.host][:status] = ch[:exit_status]
|
122
|
+
if ch[:exit_status] != 0
|
123
|
+
Multi.logger.info("execution of '#{command}' on #{ch.connection.host}
|
124
|
+
failed with return status #{ch[:exit_status].to_s}")
|
125
|
+
if results[ch.connection.host][:stdout]
|
126
|
+
Multi.logger.info("--- stdout dump ---")
|
127
|
+
Multi.logger.info(results[ch.connection.host][:stdout])
|
128
|
+
end
|
129
|
+
|
130
|
+
if results[ch.connection.host][:stderr]
|
131
|
+
Multi.logger.info("--stderr dump ---")
|
132
|
+
Multi.logger.info(results[ch.connection.host][:stderr])
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
# need to decide severity level if the command fails
|
137
|
+
end
|
138
|
+
end
|
139
|
+
main.wait # we have to wait the channel otherwise we will have void results
|
140
|
+
return results
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
end; end; end
|
data/lib/cute/net.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
module Cute
|
2
|
+
module Network
|
3
|
+
require 'socket'
|
4
|
+
|
5
|
+
def Network::port_open?(ip, port)
|
6
|
+
begin
|
7
|
+
s = TCPSocket.new(ip, port)
|
8
|
+
s.close
|
9
|
+
return true
|
10
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ETIMEDOUT
|
11
|
+
return false
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def Network::wait_open_port(host, port, timeout = 120)
|
16
|
+
def now()
|
17
|
+
return Time.now.to_f
|
18
|
+
end
|
19
|
+
bound = now() + timeout
|
20
|
+
while now() < bound do
|
21
|
+
t = now()
|
22
|
+
return true if port_open?(host, port)
|
23
|
+
dt = now() - t
|
24
|
+
sleep(0.5 - dt) if dt < 0.5
|
25
|
+
end
|
26
|
+
return false
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module Cute
|
2
|
+
module Synchronization
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
class Semaphore
|
6
|
+
def initialize(max)
|
7
|
+
@lock = Mutex.new
|
8
|
+
@cond = ConditionVariable.new
|
9
|
+
@used = 0
|
10
|
+
@max = max
|
11
|
+
end
|
12
|
+
|
13
|
+
def acquire(n = 1)
|
14
|
+
@lock.synchronize {
|
15
|
+
while (n > (@max - @used)) do
|
16
|
+
@cond.wait(@lock)
|
17
|
+
end
|
18
|
+
@used += n
|
19
|
+
}
|
20
|
+
end
|
21
|
+
|
22
|
+
def relaxed_acquire(n = 1)
|
23
|
+
taken = 0
|
24
|
+
@lock.synchronize {
|
25
|
+
while (@max == @used) do
|
26
|
+
@cond.wait(@lock)
|
27
|
+
end
|
28
|
+
taken = (n + @used) > @max ? @max - @used : n
|
29
|
+
@used += taken
|
30
|
+
}
|
31
|
+
return n - taken
|
32
|
+
end
|
33
|
+
|
34
|
+
def release(n = 1)
|
35
|
+
@lock.synchronize {
|
36
|
+
@used -= n
|
37
|
+
@cond.signal
|
38
|
+
}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class SlidingWindow
|
43
|
+
def initialize(size)
|
44
|
+
@queue = []
|
45
|
+
@lock = Mutex.new
|
46
|
+
@finished = false
|
47
|
+
@size = size
|
48
|
+
end
|
49
|
+
|
50
|
+
def add(t)
|
51
|
+
@queue << t
|
52
|
+
end
|
53
|
+
|
54
|
+
def run
|
55
|
+
tids = []
|
56
|
+
(1..@size).each {
|
57
|
+
tids << Thread.new {
|
58
|
+
while !@finished do
|
59
|
+
task = nil
|
60
|
+
@lock.synchronize {
|
61
|
+
if @queue.size > 0
|
62
|
+
task = @queue.pop
|
63
|
+
else
|
64
|
+
@finished = true
|
65
|
+
end
|
66
|
+
}
|
67
|
+
if task
|
68
|
+
if task.is_a?(Proc)
|
69
|
+
task.call
|
70
|
+
else
|
71
|
+
system(task)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
}
|
76
|
+
}
|
77
|
+
tids.each { |tid| tid.join }
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
if (__FILE__ == $0)
|
84
|
+
w = Cute::Synchronization::SlidingWindow.new(3)
|
85
|
+
(1..10).each {
|
86
|
+
w.add("sleep 1")
|
87
|
+
}
|
88
|
+
w.run
|
89
|
+
end
|
data/lib/cute/taktuk.rb
ADDED
@@ -0,0 +1,554 @@
|
|
1
|
+
module Cute
|
2
|
+
# Cute::TakTuk is a library for controlling the execution of commands in
|
3
|
+
# multiple machines using taktuk tool.
|
4
|
+
# It exposes an API similar to that of Net::SSH:Multi, making it simpler to
|
5
|
+
# adapt to scripts designed with Net::SSH::Multi.
|
6
|
+
# It simplifies the use of taktuk by automating the generation of large command line parameters.
|
7
|
+
#
|
8
|
+
# require 'cute/taktuk'
|
9
|
+
#
|
10
|
+
# results = {}
|
11
|
+
# Cute::TakTuk.start(['host1','host2','host3'],:user => "root") do |tak|
|
12
|
+
# tak.exec("df")
|
13
|
+
# results = tak.exec!("hostname")
|
14
|
+
# tak.exec("ls -l")
|
15
|
+
# tak.exec("sleep 20")
|
16
|
+
# tak.loop()
|
17
|
+
# tak.exec("tar xvf -")
|
18
|
+
# tak.input(:file => "test_file.tar")
|
19
|
+
# end
|
20
|
+
# puts results
|
21
|
+
#
|
22
|
+
# You can go directly to the documentation of useful methods such {TakTuk::TakTuk#exec exec},
|
23
|
+
# {TakTuk::TakTuk#exec! exec!}, {TakTuk::TakTuk#put put}, {TakTuk::TakTuk#input input}, etc.
|
24
|
+
# @see http://taktuk.gforge.inria.fr/.
|
25
|
+
# @see TakTuk::TakTuk TakTuk Class for more documentation.
|
26
|
+
module TakTuk
|
27
|
+
|
28
|
+
#
|
29
|
+
# Execution samples:
|
30
|
+
#
|
31
|
+
# taktuk('hostfile',:connector => 'ssh -A', :self_propagate => true).broadcast_exec['hostname'].run!
|
32
|
+
#
|
33
|
+
# taktuk(['node-1','node-2'],:dynamic => 3).broadcast_put['myfile']['dest'].run!
|
34
|
+
#
|
35
|
+
# taktuk(nodes).broadcast_exec['hostname'].seq!.broadcast_exec['df'].run!
|
36
|
+
#
|
37
|
+
# taktuk(nodes).broadcast_exec['cat - | fdisk'].seq!.broadcast_input_file['fdiskdump'].run!
|
38
|
+
#
|
39
|
+
# tak = taktuk(nodes)
|
40
|
+
# tak.broadcast_exec['hostname']
|
41
|
+
# tak.seq!.broadcast_exec['df']
|
42
|
+
# tak.streams[:output] => OutputStream.new(Template[:line,:rank]),
|
43
|
+
# tak.streams[:info] => ConnectorStream.new(Template[:command,:line])
|
44
|
+
# tak.run!
|
45
|
+
#
|
46
|
+
def self.taktuk(*args)
|
47
|
+
TakTuk.new(*args)
|
48
|
+
end
|
49
|
+
|
50
|
+
# It instantiates a new {TakTuk::TakTuk}.
|
51
|
+
# If a block is given, a {TakTuk::TakTuk} object will be yielded to the block and automatically closed when the block finishes.
|
52
|
+
# Otherwise a {TakTuk::TakTuk} object will be returned.
|
53
|
+
# @param host_list [Array] list of hosts where taktuk will execute commands on.
|
54
|
+
# @param [Hash] opts Options to be directly passed to the {TakTuk::TakTuk} object.
|
55
|
+
# @option opts [String] :user Sets the username to login into the machines.
|
56
|
+
# @option opts [String] :connector Defines the connector command used to contact the machines.
|
57
|
+
# @option opts [Array] :keys SSH keys to be used for connecting to the machines.
|
58
|
+
# @option opts [Fixnum] :port SSH port to be used for connecting to the machines.
|
59
|
+
# @option opts [String] :config SSH configuration file
|
60
|
+
# @option opts [String] :gateway Specifies a forward only node
|
61
|
+
def self.start(host_list, opts={})
|
62
|
+
taktuk_cmd = TakTuk.new(host_list, opts)
|
63
|
+
if block_given?
|
64
|
+
begin
|
65
|
+
yield taktuk_cmd
|
66
|
+
taktuk_cmd.loop unless taktuk_cmd.commands.empty?
|
67
|
+
taktuk_cmd.free! if taktuk_cmd
|
68
|
+
taktuk_cmd = nil
|
69
|
+
end
|
70
|
+
else
|
71
|
+
return taktuk_cmd
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# Parses the output generated by taktuk
|
76
|
+
# @api private
|
77
|
+
class Stream
|
78
|
+
|
79
|
+
attr_reader :types
|
80
|
+
|
81
|
+
SEPARATOR = '/'
|
82
|
+
SEPESCAPED = Regexp.escape(SEPARATOR)
|
83
|
+
IP_REGEXP = "(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}"\
|
84
|
+
"(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"
|
85
|
+
DOMAIN_REGEXP = "(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*"\
|
86
|
+
"(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])"
|
87
|
+
HOSTNAME_REGEXP = "#{IP_REGEXP}|#{DOMAIN_REGEXP}"
|
88
|
+
|
89
|
+
def initialize(types=[])
|
90
|
+
@types = types
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
def parse(string)
|
95
|
+
|
96
|
+
results = {}
|
97
|
+
if string and !string.empty?
|
98
|
+
# regexp = /^(output)#{SEPESCAPED}(#{HOSTNAME_REGEXP})#{SEPESCAPED}(.+)$/
|
99
|
+
regexp = /^(#{HOSTNAME_REGEXP})#{SEPESCAPED}(.[a-z]*)#{SEPESCAPED}(.+)$/
|
100
|
+
string.each_line do |line|
|
101
|
+
if regexp =~ line
|
102
|
+
hostname = Regexp.last_match(1)
|
103
|
+
stream_type = Regexp.last_match(2).to_sym
|
104
|
+
value_tmp = treat_value(Regexp.last_match(3))
|
105
|
+
value = value_tmp.is_i? ? value_tmp.to_i : value_tmp
|
106
|
+
results[hostname] ||= {}
|
107
|
+
if results[hostname][stream_type].nil? then
|
108
|
+
results[hostname][stream_type] = value
|
109
|
+
else
|
110
|
+
if value.is_a?(String) then
|
111
|
+
results[hostname][stream_type]+="\n" + value
|
112
|
+
else
|
113
|
+
# This is for adding status codes
|
114
|
+
results[hostname][stream_type]= [results[hostname][stream_type], value]
|
115
|
+
results[hostname][stream_type].flatten!
|
116
|
+
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
return results
|
123
|
+
|
124
|
+
end
|
125
|
+
|
126
|
+
# Return just the value 0:(.*)
|
127
|
+
def treat_value(string)
|
128
|
+
tmp = string.split(":",2)
|
129
|
+
value = tmp[1].nil? ? "" : tmp[1]
|
130
|
+
end
|
131
|
+
|
132
|
+
def to_cmd
|
133
|
+
# "\"$type#{SEPARATOR}$host#{SEPARATOR}$start_date#{SEPARATOR}$line\\n\""
|
134
|
+
# We put "0:" before $line only for performance issues when executing the regex
|
135
|
+
"\"$host#{SEPARATOR}$type#{SEPARATOR}0:$line\\n\""
|
136
|
+
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# Parses the output generated by the state template
|
141
|
+
# @api private
|
142
|
+
class StateStream < Stream
|
143
|
+
STATES = {
|
144
|
+
:error => {
|
145
|
+
3 => 'connection failed',
|
146
|
+
5 => 'connection lost',
|
147
|
+
7 => 'command failed',
|
148
|
+
9 => 'numbering update failed',
|
149
|
+
11 => 'pipe input failed',
|
150
|
+
14 => 'file reception failed',
|
151
|
+
16 => 'file send failed',
|
152
|
+
17 => 'invalid target',
|
153
|
+
18 => 'no target',
|
154
|
+
20 => 'invalid destination',
|
155
|
+
21 => 'destination not available anymore',
|
156
|
+
},
|
157
|
+
:progress => {
|
158
|
+
0 => 'taktuk is ready',
|
159
|
+
1 => 'taktuk is numbered',
|
160
|
+
4 => 'connection initialized',
|
161
|
+
6 => 'command started',
|
162
|
+
10 => 'pipe input started',
|
163
|
+
13 => 'file reception started',
|
164
|
+
},
|
165
|
+
:done => {
|
166
|
+
2 => 'taktuk terminated',
|
167
|
+
8 => 'command terminated',
|
168
|
+
12 => 'pipe input terminated',
|
169
|
+
15 => 'file reception terminated',
|
170
|
+
19 => 'message delivered',
|
171
|
+
}
|
172
|
+
}
|
173
|
+
|
174
|
+
def initialize(template)
|
175
|
+
super(:state,template)
|
176
|
+
end
|
177
|
+
|
178
|
+
# type can be :error, :progress or :done
|
179
|
+
def self.check?(type,state)
|
180
|
+
return nil unless STATES[type]
|
181
|
+
state = state.strip
|
182
|
+
|
183
|
+
begin
|
184
|
+
nb = Integer(state)
|
185
|
+
STATES[type].keys.include?(nb)
|
186
|
+
rescue
|
187
|
+
STATES[type].values.include?(state.downcase!)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def self.errmsg(nb)
|
192
|
+
STATES.each_value do |typeval|
|
193
|
+
return typeval[nb] if typeval[nb]
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
# Validates taktuk options
|
199
|
+
# @api private
|
200
|
+
class Options < Hash
|
201
|
+
TAKTUK_VALID = [
|
202
|
+
'begin-group', 'connector', 'dynamic', 'end-group', 'machines-file',
|
203
|
+
'login', 'machine', 'self-propagate', 'dont-self-propagate',
|
204
|
+
'args-file', 'gateway', 'perl-interpreter', 'localhost',
|
205
|
+
'send-files', 'taktuk-command', 'path-value', 'command-separator',
|
206
|
+
'escape-character', 'option-separator', 'output-redirect',
|
207
|
+
'worksteal-behavior', 'time-granularity', 'no-numbering', 'timeout',
|
208
|
+
'cache-limit', 'window','window-adaptation','not-root','debug'
|
209
|
+
]
|
210
|
+
WRAPPER_VALID = [ 'streams', 'port', 'keys', 'user', 'config' ] # user is an alias for login
|
211
|
+
|
212
|
+
def check(optname)
|
213
|
+
ret = optname.to_s.gsub(/_/,'-').strip
|
214
|
+
raise ArgumentError.new("Invalid TakTuk option '--#{ret}'") unless TAKTUK_VALID.include?(ret)
|
215
|
+
ret
|
216
|
+
end
|
217
|
+
|
218
|
+
def to_cmd
|
219
|
+
|
220
|
+
self[:login] = self[:user] if keys.include?(:user)
|
221
|
+
self.keys.inject([]) do |ret,opt|
|
222
|
+
if not WRAPPER_VALID.include?(opt.to_s) then
|
223
|
+
ret << "--#{check(opt)}"
|
224
|
+
if self[opt]
|
225
|
+
if self[opt].is_a?(String)
|
226
|
+
ret << self[opt] unless self[opt].empty?
|
227
|
+
else
|
228
|
+
ret << self[opt].to_s
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
ret
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
# Generates a taktuk CLI compatible host list
|
238
|
+
# @api private
|
239
|
+
class Hostlist
|
240
|
+
def initialize(hostlist)
|
241
|
+
@hostlist=hostlist
|
242
|
+
end
|
243
|
+
|
244
|
+
def free
|
245
|
+
@hostlist = nil
|
246
|
+
end
|
247
|
+
|
248
|
+
def exclude(node)
|
249
|
+
@hostlist.remove(node) if @hostlist.is_a?(Array)
|
250
|
+
end
|
251
|
+
|
252
|
+
def to_cmd
|
253
|
+
ret = []
|
254
|
+
if @hostlist.is_a?(Array)
|
255
|
+
@hostlist.each do |host|
|
256
|
+
ret << '-m'
|
257
|
+
ret << host
|
258
|
+
end
|
259
|
+
elsif @hostlist.is_a?(String)
|
260
|
+
ret << '-f'
|
261
|
+
ret << @hostlist
|
262
|
+
end
|
263
|
+
ret
|
264
|
+
end
|
265
|
+
|
266
|
+
def to_a
|
267
|
+
if @hostlist.is_a?(Array)
|
268
|
+
@hostlist
|
269
|
+
elsif @hostlist.is_a?(String)
|
270
|
+
raise "Hostfile does not exist" unless File.exist?(@hostlist)
|
271
|
+
File.read(@hostlist).split("\n").uniq
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
276
|
+
# Validates the commands accepted by taktuk
|
277
|
+
# @api private
|
278
|
+
class Commands < Array
|
279
|
+
TOKENS=[
|
280
|
+
'broadcast', 'downcast', 'exec', 'get', 'put', 'input', 'data',
|
281
|
+
'file', 'pipe', 'close', 'line', 'target', 'kill', 'message',
|
282
|
+
'network', 'state', 'cancel', 'renumber', 'update', 'option',
|
283
|
+
'synchronize', 'taktuk_perl', 'quit', 'wait', 'reduce'
|
284
|
+
]
|
285
|
+
|
286
|
+
def <<(val)
|
287
|
+
raise ArgumentError.new("'Invalid TakTuk command '#{val}'") unless check(val)
|
288
|
+
super(val)
|
289
|
+
end
|
290
|
+
|
291
|
+
def check(val)
|
292
|
+
if val =~ /^-?\[.*-?\]$|^;$/
|
293
|
+
true
|
294
|
+
elsif val.nil? or val.empty?
|
295
|
+
false
|
296
|
+
else
|
297
|
+
tmp = val.split(' ',2)
|
298
|
+
return false unless valid?(tmp[0])
|
299
|
+
if !tmp[1].nil? and !tmp[1].empty?
|
300
|
+
check(tmp[1])
|
301
|
+
else
|
302
|
+
true
|
303
|
+
end
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
def valid?(value)
|
308
|
+
TOKENS.each do |token|
|
309
|
+
return true if token =~ /^#{Regexp.escape(value)}.*$/
|
310
|
+
end
|
311
|
+
return false
|
312
|
+
end
|
313
|
+
|
314
|
+
def to_cmd
|
315
|
+
self.inject([]) do |ret,val|
|
316
|
+
if val =~ /^\[(.*)\]$/
|
317
|
+
ret += ['[',Regexp.last_match(1).strip,']']
|
318
|
+
else
|
319
|
+
ret += val.split(' ')
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
# This class wraps the command TakTuk and generates automatically
|
326
|
+
# the long CLI options for taktuk command.
|
327
|
+
class TakTuk
|
328
|
+
attr_accessor :streams,:binary
|
329
|
+
attr_reader :stdout,:stderr,:status, :args, :exec_cmd, :commands
|
330
|
+
|
331
|
+
VALID_STREAMS = [:output, :error, :status, :connector, :state, :info, :message, :taktuk ]
|
332
|
+
|
333
|
+
def initialize(hostlist,options = {:connector => 'ssh'})
|
334
|
+
raise ArgumentError.new("options parameter has to be a hash") unless options.is_a?(Hash)
|
335
|
+
|
336
|
+
@binary = 'taktuk'
|
337
|
+
@options = Options[options.merge({ :streams => [:output, :error, :status ]})] if options[:streams].nil?
|
338
|
+
@options.merge!({:connector => 'ssh'}) if options[:connector].nil?
|
339
|
+
@streams = Stream.new(@options[:streams])
|
340
|
+
# @streams = Stream.new([:output,:error,:status, :state])
|
341
|
+
|
342
|
+
@hostlist = Hostlist.new(hostlist)
|
343
|
+
@commands = Commands.new
|
344
|
+
|
345
|
+
@args = nil
|
346
|
+
@stdout = nil
|
347
|
+
@stderr = nil
|
348
|
+
@status = nil
|
349
|
+
|
350
|
+
@exec_cmd = nil
|
351
|
+
@curthread = nil
|
352
|
+
@connector = @options[:connector]
|
353
|
+
end
|
354
|
+
|
355
|
+
def run!(opts = {})
|
356
|
+
@curthread = Thread.current
|
357
|
+
@args = []
|
358
|
+
@args += @options.to_cmd
|
359
|
+
|
360
|
+
@streams.types.each{ |name|
|
361
|
+
@args << '-o'
|
362
|
+
@args << "#{name.to_s}=#{@streams.to_cmd}"
|
363
|
+
}
|
364
|
+
|
365
|
+
connector = build_connector
|
366
|
+
@args += ["--connector", "#{connector}"] unless connector.nil?
|
367
|
+
|
368
|
+
@args += @hostlist.to_cmd
|
369
|
+
@args += @commands.to_cmd
|
370
|
+
|
371
|
+
hosts = @hostlist.to_a
|
372
|
+
outputs_size = opts[:outputs_size] || 0
|
373
|
+
@exec_cmd = Cute::Execute[@binary,*@args].run!(
|
374
|
+
:stdout_size => outputs_size * hosts.size,
|
375
|
+
:stderr_size => outputs_size * hosts.size,
|
376
|
+
:stdin => false
|
377
|
+
)
|
378
|
+
@status, @stdout, @stderr, emptypipes = @exec_cmd.wait({:checkstatus=>false})
|
379
|
+
|
380
|
+
unless @status.success?
|
381
|
+
@curthread = nil
|
382
|
+
return false
|
383
|
+
end
|
384
|
+
|
385
|
+
unless emptypipes
|
386
|
+
@curthread = nil
|
387
|
+
@stderr = "Too much data on the TakTuk command's stdout/stderr"
|
388
|
+
return false
|
389
|
+
end
|
390
|
+
|
391
|
+
results = @streams.parse(@stdout)
|
392
|
+
@curthread = nil
|
393
|
+
|
394
|
+
results
|
395
|
+
end
|
396
|
+
|
397
|
+
# It executes the commands so far stored in the @commands variable
|
398
|
+
# and reinitialize the variable for post utilization.
|
399
|
+
def loop ()
|
400
|
+
run!()
|
401
|
+
$stdout.print(@stdout)
|
402
|
+
$stderr.print(@stderr)
|
403
|
+
@commands = Commands.new
|
404
|
+
end
|
405
|
+
|
406
|
+
def kill!()
|
407
|
+
unless @exec.nil?
|
408
|
+
@exec.kill
|
409
|
+
@exec = nil
|
410
|
+
end
|
411
|
+
free!()
|
412
|
+
end
|
413
|
+
|
414
|
+
def free!()
|
415
|
+
@binary = nil
|
416
|
+
@options = nil
|
417
|
+
# if @streams
|
418
|
+
# @streams.each_value do |stream|
|
419
|
+
# stream.free if stream
|
420
|
+
# stream = nil
|
421
|
+
# end
|
422
|
+
# end
|
423
|
+
@hostlist.free if @hostlist
|
424
|
+
@hostlist = nil
|
425
|
+
@commands = nil
|
426
|
+
@args = nil
|
427
|
+
@stdout = nil
|
428
|
+
@stderr = nil
|
429
|
+
@status = nil
|
430
|
+
@exec = nil
|
431
|
+
@curthread = nil
|
432
|
+
end
|
433
|
+
|
434
|
+
def raw!(string)
|
435
|
+
@commands << string.strip
|
436
|
+
self
|
437
|
+
end
|
438
|
+
|
439
|
+
# It executes a command on multiple hosts.
|
440
|
+
# All output is printed via *stdout* and *stderr*.
|
441
|
+
# Note that this method returns immediately,
|
442
|
+
# and requires a call to the loop method in order
|
443
|
+
# for the command to actually execute.
|
444
|
+
# The execution is done by TakTuk using broadcast exec.
|
445
|
+
# @param [String] cmd Command to execute.
|
446
|
+
#
|
447
|
+
# = Example
|
448
|
+
#
|
449
|
+
# tak.exec("hostname")
|
450
|
+
# tak.exec("mkdir ~/test")
|
451
|
+
# tak.loop() # to trigger the execution of commands
|
452
|
+
def exec(cmd)
|
453
|
+
mode = "broadcast"
|
454
|
+
@commands << "#{mode} exec"
|
455
|
+
@commands << "[ #{cmd} ]"
|
456
|
+
@commands << ';' # TakTuk command separator
|
457
|
+
end
|
458
|
+
|
459
|
+
# It transfers a file to all the machines in parallel.
|
460
|
+
# @param [String] source Source path for the file to be transfer
|
461
|
+
# @param [String] dest Destination path for the file to be transfer
|
462
|
+
#
|
463
|
+
# = Example
|
464
|
+
#
|
465
|
+
# tak.put("hosts.allow_template", "/etc/hosts.allow")
|
466
|
+
#
|
467
|
+
def put(source,dest)
|
468
|
+
mode = "broadcast"
|
469
|
+
@commands << "#{mode} put"
|
470
|
+
@commands << "[ #{source} ]"
|
471
|
+
@commands << "[ #{dest} ]"
|
472
|
+
@commands << ';' # TakTuk command separator
|
473
|
+
end
|
474
|
+
|
475
|
+
|
476
|
+
# It executes a command on multiple hosts capturing the output,
|
477
|
+
# and other information related with the execution.
|
478
|
+
# It blocks until the command finishes.
|
479
|
+
# @param [String] cmd Command to be executed
|
480
|
+
# @return [Hash] Result data structure
|
481
|
+
#
|
482
|
+
# = Example
|
483
|
+
#
|
484
|
+
# tak.exec!("uname -r") #=> {"node2"=>{:output=>"3.2.0-4-amd64", :status=>0}, "node3"=>{:output=>"3.2.0-4-amd64", :status=>0}, ...}
|
485
|
+
#
|
486
|
+
def exec!(cmd)
|
487
|
+
loop() unless @commands.empty?
|
488
|
+
exec(cmd)
|
489
|
+
results = run!()
|
490
|
+
@commands = Commands.new
|
491
|
+
return results
|
492
|
+
end
|
493
|
+
|
494
|
+
# Manages the taktuk command input
|
495
|
+
# @param [Hash] opts Options for the type of data
|
496
|
+
# @option opts [String] :data Raw data to be used as the input of a command
|
497
|
+
# @option opts [String] :filename a file to be used as the input of a command
|
498
|
+
#
|
499
|
+
# = Example
|
500
|
+
#
|
501
|
+
# tak.exec("wc -w")
|
502
|
+
# tak.input(:data => "data data data data")
|
503
|
+
#
|
504
|
+
# tak.exec("tar xvf -")
|
505
|
+
# tak.input(:file => "test_file.tar")
|
506
|
+
def input(opts = {})
|
507
|
+
mode = "broadcast"
|
508
|
+
@commands << "#{mode} input #{opts.keys.first.to_s}"
|
509
|
+
@commands << "[ #{opts.values.first} ]"
|
510
|
+
@commands << ';'
|
511
|
+
end
|
512
|
+
|
513
|
+
|
514
|
+
def [](command,prefix='[',suffix=']')
|
515
|
+
@commands << "#{prefix} #{command} #{suffix}"
|
516
|
+
self
|
517
|
+
end
|
518
|
+
|
519
|
+
def method_missing(meth,*args)
|
520
|
+
@commands << (meth.to_s.gsub(/_/,' ').strip.downcase)
|
521
|
+
args.each do |arg|
|
522
|
+
@commands.push(arg.strip.downcase)
|
523
|
+
end
|
524
|
+
self
|
525
|
+
end
|
526
|
+
|
527
|
+
alias close free!
|
528
|
+
|
529
|
+
private
|
530
|
+
# It builds a custom connector for TakTuk
|
531
|
+
def build_connector()
|
532
|
+
ssh_options = [:keys, :port, :config]
|
533
|
+
connector = @connector
|
534
|
+
if @options.keys.map{ |opt| ssh_options.include?(opt)}.any?
|
535
|
+
connector += " -p #{@options[:port]}" if @options[:port]
|
536
|
+
if @options[:keys]
|
537
|
+
keys = @options[:keys].is_a?(Array) ? @options[:keys].first : @options[:keys]
|
538
|
+
connector += " -i #{keys}"
|
539
|
+
end
|
540
|
+
connector += " -F #{@options[:config]}" if @options[:config]
|
541
|
+
end
|
542
|
+
return connector
|
543
|
+
end
|
544
|
+
|
545
|
+
# It is used to separate the commands, they will run in parallel.
|
546
|
+
def seq!
|
547
|
+
@commands << ';'
|
548
|
+
self
|
549
|
+
end
|
550
|
+
|
551
|
+
end
|
552
|
+
|
553
|
+
end
|
554
|
+
end
|