ruby-cute 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.yardopts +2 -0
- data/Gemfile +6 -0
- data/README.md +137 -6
- data/Rakefile +48 -0
- data/bin/cute +22 -0
- data/debian/changelog +5 -0
- data/debian/compat +1 -0
- data/debian/control +15 -0
- data/debian/copyright +33 -0
- data/debian/ruby-cute.docs +2 -0
- data/debian/ruby-tests.rb +2 -0
- data/debian/rules +19 -0
- data/debian/source/format +1 -0
- data/debian/watch +2 -0
- data/examples/distem-bootstrap +516 -0
- data/examples/g5k_exp1.rb +41 -0
- data/examples/g5k_exp_virt.rb +129 -0
- data/lib/cute.rb +7 -2
- data/lib/cute/bash.rb +337 -0
- data/lib/cute/configparser.rb +404 -0
- data/lib/cute/execute.rb +272 -0
- data/lib/cute/extensions.rb +38 -0
- data/lib/cute/g5k_api.rb +1190 -0
- data/lib/cute/net-ssh.rb +144 -0
- data/lib/cute/net.rb +29 -0
- data/lib/cute/synchronization.rb +89 -0
- data/lib/cute/taktuk.rb +554 -0
- data/lib/cute/version.rb +3 -0
- data/ruby-cute.gemspec +32 -0
- data/spec/extensions_spec.rb +17 -0
- data/spec/g5k_api_spec.rb +192 -0
- data/spec/spec_helper.rb +66 -0
- data/spec/taktuk_spec.rb +129 -0
- data/test/test_bash.rb +71 -0
- metadata +204 -47
data/lib/cute/net-ssh.rb
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'net/ssh/multi'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Net; module SSH
|
5
|
+
|
6
|
+
# The Net::SSH::Multi aims at executing commands in parallel over a set of machines using the SSH protocol.
|
7
|
+
# One of the advantage of this module over {Cute::TakTuk::TakTuk TakTuk} is that it allows to create groups, for example:
|
8
|
+
#
|
9
|
+
# Net::SSH::Multi.start do |session|
|
10
|
+
#
|
11
|
+
# session.group :coord do
|
12
|
+
# session.use("root@#{coordinator}")
|
13
|
+
# end
|
14
|
+
#
|
15
|
+
# session.group :nodes do
|
16
|
+
# nodelist.each{ |node| session.use("root@#{node}")}
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
# # test connection
|
20
|
+
# session.with(:coord).exec! "hostname"
|
21
|
+
# session.with(:nodes).exec! "hostname"
|
22
|
+
#
|
23
|
+
# # Check nfs paths
|
24
|
+
# tmp = session.exec! "ls -a #{ENV['HOME']}"
|
25
|
+
#
|
26
|
+
# # generating ssh password less connection
|
27
|
+
# session.exec! "cat .ssh/id_rsa.pub >> .ssh/authorized_keys"
|
28
|
+
# end
|
29
|
+
#
|
30
|
+
# However, with large set of nodes SSH it is limited an inefficient,
|
31
|
+
# for those cases the best option will be {Cute::TakTuk::TakTuk TakTuk}.
|
32
|
+
# For complete documentation please take a look at
|
33
|
+
# {http://net-ssh.github.io/net-ssh-multi/ Net::SSH::Multi}.
|
34
|
+
# One of the disadvantages of {http://net-ssh.github.io/net-ssh-multi/ Net::SSH::Multi} is that
|
35
|
+
# it does not allow to capture the output (stdout, stderr and status) of executed commands.
|
36
|
+
# Ruby-Cute ships a monkey patch that extends the aforementioned module by adding the method
|
37
|
+
# {Net::SSH::Multi::SessionActions#exec! exec!}
|
38
|
+
# which blocks until the command finishes and captures the output (stdout, stderr and status).
|
39
|
+
#
|
40
|
+
# require 'cute/net-ssh'
|
41
|
+
#
|
42
|
+
# results = {}
|
43
|
+
# Net::SSH::Multi.start do |session|
|
44
|
+
#
|
45
|
+
# # define the servers we want to use
|
46
|
+
# session.use 'user1@host1'
|
47
|
+
# session.use 'user2@host2'
|
48
|
+
#
|
49
|
+
# session.exec "uptime"
|
50
|
+
# session.exec "df"
|
51
|
+
# # execute command, blocks and capture the output
|
52
|
+
# results = session.exec! "date"
|
53
|
+
# # execute commands on a subset of servers
|
54
|
+
# session.exec "hostname"
|
55
|
+
# end
|
56
|
+
# puts results #=> {"node3"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0},
|
57
|
+
# # "node1"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0}, ...}
|
58
|
+
#
|
59
|
+
module Multi
|
60
|
+
|
61
|
+
# sets logger to be used by net-ssh-multi module
|
62
|
+
def self.logger= v
|
63
|
+
@logger = v
|
64
|
+
end
|
65
|
+
|
66
|
+
# @return logger
|
67
|
+
def self.logger
|
68
|
+
if @logger.nil?
|
69
|
+
@logger = Logger.new(STDOUT)
|
70
|
+
logger.level = Logger::INFO
|
71
|
+
end
|
72
|
+
@logger
|
73
|
+
end
|
74
|
+
|
75
|
+
module SessionActions
|
76
|
+
|
77
|
+
# Monkey patch that adds the exec! method.
|
78
|
+
# It executes a command on multiple hosts capturing their associated output (stdout, stderr and status).
|
79
|
+
# It blocks until the command finishes returning the resulting output as a Hash.
|
80
|
+
# It uses a logger for debugging purposes.
|
81
|
+
# @see http://net-ssh.github.io/net-ssh-multi/classes/Net/SSH/Multi/SessionActions.html More information about exec method.
|
82
|
+
# @return [Hash] result Hash stdout, stderr and status of executed commands
|
83
|
+
#
|
84
|
+
# = Example
|
85
|
+
#
|
86
|
+
# session.exec!("date") #=> {"node3"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0},
|
87
|
+
# # "node1"=>{:stdout=>"Wed Mar 11 12:38:11 UTC 2015", :status=>0}, ...}
|
88
|
+
#
|
89
|
+
# session.exec!("cmd") #=> {"node4"=>{:stderr=>"bash: cmd: command not found", :status=>127},
|
90
|
+
# # "node3"=>{:stderr=>"bash: cmd: command not found", :status=>127}, ...}
|
91
|
+
#
|
92
|
+
def exec!(command, &block)
|
93
|
+
|
94
|
+
results = {}
|
95
|
+
|
96
|
+
main =open_channel do |channel|
|
97
|
+
channel.exec(command) do |ch, success|
|
98
|
+
raise "could not execute command: #{command.inspect} (#{ch[:host]})" unless success
|
99
|
+
Multi.logger.debug("Executing #{command} on [#{ch.connection.host}]")
|
100
|
+
|
101
|
+
results[ch.connection.host] ||= {}
|
102
|
+
|
103
|
+
channel.on_data do |ch, data|
|
104
|
+
if block
|
105
|
+
block.call(ch, :stdout, data)
|
106
|
+
else
|
107
|
+
results[ch.connection.host][:stdout] = data.strip
|
108
|
+
Multi.logger.debug("[#{ch.connection.host}] #{data.strip}")
|
109
|
+
end
|
110
|
+
end
|
111
|
+
channel.on_extended_data do |ch, type, data|
|
112
|
+
if block
|
113
|
+
block.call(ch, :stderr, data)
|
114
|
+
else
|
115
|
+
results[ch.connection.host][:stderr] = data.strip
|
116
|
+
Multi.logger.debug("[#{ch.connection.host}] #{data.strip}")
|
117
|
+
end
|
118
|
+
end
|
119
|
+
channel.on_request("exit-status") do |ch, data|
|
120
|
+
ch[:exit_status] = data.read_long
|
121
|
+
results[ch.connection.host][:status] = ch[:exit_status]
|
122
|
+
if ch[:exit_status] != 0
|
123
|
+
Multi.logger.info("execution of '#{command}' on #{ch.connection.host}
|
124
|
+
failed with return status #{ch[:exit_status].to_s}")
|
125
|
+
if results[ch.connection.host][:stdout]
|
126
|
+
Multi.logger.info("--- stdout dump ---")
|
127
|
+
Multi.logger.info(results[ch.connection.host][:stdout])
|
128
|
+
end
|
129
|
+
|
130
|
+
if results[ch.connection.host][:stderr]
|
131
|
+
Multi.logger.info("--stderr dump ---")
|
132
|
+
Multi.logger.info(results[ch.connection.host][:stderr])
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
# need to decide severity level if the command fails
|
137
|
+
end
|
138
|
+
end
|
139
|
+
main.wait # we have to wait the channel otherwise we will have void results
|
140
|
+
return results
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
end; end; end
|
data/lib/cute/net.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
module Cute
|
2
|
+
module Network
|
3
|
+
require 'socket'
|
4
|
+
|
5
|
+
def Network::port_open?(ip, port)
|
6
|
+
begin
|
7
|
+
s = TCPSocket.new(ip, port)
|
8
|
+
s.close
|
9
|
+
return true
|
10
|
+
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ETIMEDOUT
|
11
|
+
return false
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def Network::wait_open_port(host, port, timeout = 120)
|
16
|
+
def now()
|
17
|
+
return Time.now.to_f
|
18
|
+
end
|
19
|
+
bound = now() + timeout
|
20
|
+
while now() < bound do
|
21
|
+
t = now()
|
22
|
+
return true if port_open?(host, port)
|
23
|
+
dt = now() - t
|
24
|
+
sleep(0.5 - dt) if dt < 0.5
|
25
|
+
end
|
26
|
+
return false
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module Cute
|
2
|
+
module Synchronization
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
class Semaphore
|
6
|
+
def initialize(max)
|
7
|
+
@lock = Mutex.new
|
8
|
+
@cond = ConditionVariable.new
|
9
|
+
@used = 0
|
10
|
+
@max = max
|
11
|
+
end
|
12
|
+
|
13
|
+
def acquire(n = 1)
|
14
|
+
@lock.synchronize {
|
15
|
+
while (n > (@max - @used)) do
|
16
|
+
@cond.wait(@lock)
|
17
|
+
end
|
18
|
+
@used += n
|
19
|
+
}
|
20
|
+
end
|
21
|
+
|
22
|
+
def relaxed_acquire(n = 1)
|
23
|
+
taken = 0
|
24
|
+
@lock.synchronize {
|
25
|
+
while (@max == @used) do
|
26
|
+
@cond.wait(@lock)
|
27
|
+
end
|
28
|
+
taken = (n + @used) > @max ? @max - @used : n
|
29
|
+
@used += taken
|
30
|
+
}
|
31
|
+
return n - taken
|
32
|
+
end
|
33
|
+
|
34
|
+
def release(n = 1)
|
35
|
+
@lock.synchronize {
|
36
|
+
@used -= n
|
37
|
+
@cond.signal
|
38
|
+
}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class SlidingWindow
|
43
|
+
def initialize(size)
|
44
|
+
@queue = []
|
45
|
+
@lock = Mutex.new
|
46
|
+
@finished = false
|
47
|
+
@size = size
|
48
|
+
end
|
49
|
+
|
50
|
+
def add(t)
|
51
|
+
@queue << t
|
52
|
+
end
|
53
|
+
|
54
|
+
def run
|
55
|
+
tids = []
|
56
|
+
(1..@size).each {
|
57
|
+
tids << Thread.new {
|
58
|
+
while !@finished do
|
59
|
+
task = nil
|
60
|
+
@lock.synchronize {
|
61
|
+
if @queue.size > 0
|
62
|
+
task = @queue.pop
|
63
|
+
else
|
64
|
+
@finished = true
|
65
|
+
end
|
66
|
+
}
|
67
|
+
if task
|
68
|
+
if task.is_a?(Proc)
|
69
|
+
task.call
|
70
|
+
else
|
71
|
+
system(task)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
}
|
76
|
+
}
|
77
|
+
tids.each { |tid| tid.join }
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
if (__FILE__ == $0)
|
84
|
+
w = Cute::Synchronization::SlidingWindow.new(3)
|
85
|
+
(1..10).each {
|
86
|
+
w.add("sleep 1")
|
87
|
+
}
|
88
|
+
w.run
|
89
|
+
end
|
data/lib/cute/taktuk.rb
ADDED
@@ -0,0 +1,554 @@
|
|
1
|
+
module Cute
|
2
|
+
# Cute::TakTuk is a library for controlling the execution of commands in
|
3
|
+
# multiple machines using taktuk tool.
|
4
|
+
# It exposes an API similar to that of Net::SSH:Multi, making it simpler to
|
5
|
+
# adapt to scripts designed with Net::SSH::Multi.
|
6
|
+
# It simplifies the use of taktuk by automating the generation of large command line parameters.
|
7
|
+
#
|
8
|
+
# require 'cute/taktuk'
|
9
|
+
#
|
10
|
+
# results = {}
|
11
|
+
# Cute::TakTuk.start(['host1','host2','host3'],:user => "root") do |tak|
|
12
|
+
# tak.exec("df")
|
13
|
+
# results = tak.exec!("hostname")
|
14
|
+
# tak.exec("ls -l")
|
15
|
+
# tak.exec("sleep 20")
|
16
|
+
# tak.loop()
|
17
|
+
# tak.exec("tar xvf -")
|
18
|
+
# tak.input(:file => "test_file.tar")
|
19
|
+
# end
|
20
|
+
# puts results
|
21
|
+
#
|
22
|
+
# You can go directly to the documentation of useful methods such {TakTuk::TakTuk#exec exec},
|
23
|
+
# {TakTuk::TakTuk#exec! exec!}, {TakTuk::TakTuk#put put}, {TakTuk::TakTuk#input input}, etc.
|
24
|
+
# @see http://taktuk.gforge.inria.fr/.
|
25
|
+
# @see TakTuk::TakTuk TakTuk Class for more documentation.
|
26
|
+
module TakTuk
|
27
|
+
|
28
|
+
#
|
29
|
+
# Execution samples:
|
30
|
+
#
|
31
|
+
# taktuk('hostfile',:connector => 'ssh -A', :self_propagate => true).broadcast_exec['hostname'].run!
|
32
|
+
#
|
33
|
+
# taktuk(['node-1','node-2'],:dynamic => 3).broadcast_put['myfile']['dest'].run!
|
34
|
+
#
|
35
|
+
# taktuk(nodes).broadcast_exec['hostname'].seq!.broadcast_exec['df'].run!
|
36
|
+
#
|
37
|
+
# taktuk(nodes).broadcast_exec['cat - | fdisk'].seq!.broadcast_input_file['fdiskdump'].run!
|
38
|
+
#
|
39
|
+
# tak = taktuk(nodes)
|
40
|
+
# tak.broadcast_exec['hostname']
|
41
|
+
# tak.seq!.broadcast_exec['df']
|
42
|
+
# tak.streams[:output] => OutputStream.new(Template[:line,:rank]),
|
43
|
+
# tak.streams[:info] => ConnectorStream.new(Template[:command,:line])
|
44
|
+
# tak.run!
|
45
|
+
#
|
46
|
+
def self.taktuk(*args)
|
47
|
+
TakTuk.new(*args)
|
48
|
+
end
|
49
|
+
|
50
|
+
# It instantiates a new {TakTuk::TakTuk}.
|
51
|
+
# If a block is given, a {TakTuk::TakTuk} object will be yielded to the block and automatically closed when the block finishes.
|
52
|
+
# Otherwise a {TakTuk::TakTuk} object will be returned.
|
53
|
+
# @param host_list [Array] list of hosts where taktuk will execute commands on.
|
54
|
+
# @param [Hash] opts Options to be directly passed to the {TakTuk::TakTuk} object.
|
55
|
+
# @option opts [String] :user Sets the username to login into the machines.
|
56
|
+
# @option opts [String] :connector Defines the connector command used to contact the machines.
|
57
|
+
# @option opts [Array] :keys SSH keys to be used for connecting to the machines.
|
58
|
+
# @option opts [Fixnum] :port SSH port to be used for connecting to the machines.
|
59
|
+
# @option opts [String] :config SSH configuration file
|
60
|
+
# @option opts [String] :gateway Specifies a forward only node
|
61
|
+
def self.start(host_list, opts={})
|
62
|
+
taktuk_cmd = TakTuk.new(host_list, opts)
|
63
|
+
if block_given?
|
64
|
+
begin
|
65
|
+
yield taktuk_cmd
|
66
|
+
taktuk_cmd.loop unless taktuk_cmd.commands.empty?
|
67
|
+
taktuk_cmd.free! if taktuk_cmd
|
68
|
+
taktuk_cmd = nil
|
69
|
+
end
|
70
|
+
else
|
71
|
+
return taktuk_cmd
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# Parses the output generated by taktuk
|
76
|
+
# @api private
|
77
|
+
class Stream
|
78
|
+
|
79
|
+
attr_reader :types
|
80
|
+
|
81
|
+
SEPARATOR = '/'
|
82
|
+
SEPESCAPED = Regexp.escape(SEPARATOR)
|
83
|
+
IP_REGEXP = "(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}"\
|
84
|
+
"(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"
|
85
|
+
DOMAIN_REGEXP = "(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*"\
|
86
|
+
"(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])"
|
87
|
+
HOSTNAME_REGEXP = "#{IP_REGEXP}|#{DOMAIN_REGEXP}"
|
88
|
+
|
89
|
+
def initialize(types=[])
|
90
|
+
@types = types
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
def parse(string)
|
95
|
+
|
96
|
+
results = {}
|
97
|
+
if string and !string.empty?
|
98
|
+
# regexp = /^(output)#{SEPESCAPED}(#{HOSTNAME_REGEXP})#{SEPESCAPED}(.+)$/
|
99
|
+
regexp = /^(#{HOSTNAME_REGEXP})#{SEPESCAPED}(.[a-z]*)#{SEPESCAPED}(.+)$/
|
100
|
+
string.each_line do |line|
|
101
|
+
if regexp =~ line
|
102
|
+
hostname = Regexp.last_match(1)
|
103
|
+
stream_type = Regexp.last_match(2).to_sym
|
104
|
+
value_tmp = treat_value(Regexp.last_match(3))
|
105
|
+
value = value_tmp.is_i? ? value_tmp.to_i : value_tmp
|
106
|
+
results[hostname] ||= {}
|
107
|
+
if results[hostname][stream_type].nil? then
|
108
|
+
results[hostname][stream_type] = value
|
109
|
+
else
|
110
|
+
if value.is_a?(String) then
|
111
|
+
results[hostname][stream_type]+="\n" + value
|
112
|
+
else
|
113
|
+
# This is for adding status codes
|
114
|
+
results[hostname][stream_type]= [results[hostname][stream_type], value]
|
115
|
+
results[hostname][stream_type].flatten!
|
116
|
+
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
return results
|
123
|
+
|
124
|
+
end
|
125
|
+
|
126
|
+
# Return just the value 0:(.*)
|
127
|
+
def treat_value(string)
|
128
|
+
tmp = string.split(":",2)
|
129
|
+
value = tmp[1].nil? ? "" : tmp[1]
|
130
|
+
end
|
131
|
+
|
132
|
+
def to_cmd
|
133
|
+
# "\"$type#{SEPARATOR}$host#{SEPARATOR}$start_date#{SEPARATOR}$line\\n\""
|
134
|
+
# We put "0:" before $line only for performance issues when executing the regex
|
135
|
+
"\"$host#{SEPARATOR}$type#{SEPARATOR}0:$line\\n\""
|
136
|
+
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# Parses the output generated by the state template
|
141
|
+
# @api private
|
142
|
+
class StateStream < Stream
|
143
|
+
STATES = {
|
144
|
+
:error => {
|
145
|
+
3 => 'connection failed',
|
146
|
+
5 => 'connection lost',
|
147
|
+
7 => 'command failed',
|
148
|
+
9 => 'numbering update failed',
|
149
|
+
11 => 'pipe input failed',
|
150
|
+
14 => 'file reception failed',
|
151
|
+
16 => 'file send failed',
|
152
|
+
17 => 'invalid target',
|
153
|
+
18 => 'no target',
|
154
|
+
20 => 'invalid destination',
|
155
|
+
21 => 'destination not available anymore',
|
156
|
+
},
|
157
|
+
:progress => {
|
158
|
+
0 => 'taktuk is ready',
|
159
|
+
1 => 'taktuk is numbered',
|
160
|
+
4 => 'connection initialized',
|
161
|
+
6 => 'command started',
|
162
|
+
10 => 'pipe input started',
|
163
|
+
13 => 'file reception started',
|
164
|
+
},
|
165
|
+
:done => {
|
166
|
+
2 => 'taktuk terminated',
|
167
|
+
8 => 'command terminated',
|
168
|
+
12 => 'pipe input terminated',
|
169
|
+
15 => 'file reception terminated',
|
170
|
+
19 => 'message delivered',
|
171
|
+
}
|
172
|
+
}
|
173
|
+
|
174
|
+
def initialize(template)
|
175
|
+
super(:state,template)
|
176
|
+
end
|
177
|
+
|
178
|
+
# type can be :error, :progress or :done
|
179
|
+
def self.check?(type,state)
|
180
|
+
return nil unless STATES[type]
|
181
|
+
state = state.strip
|
182
|
+
|
183
|
+
begin
|
184
|
+
nb = Integer(state)
|
185
|
+
STATES[type].keys.include?(nb)
|
186
|
+
rescue
|
187
|
+
STATES[type].values.include?(state.downcase!)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def self.errmsg(nb)
|
192
|
+
STATES.each_value do |typeval|
|
193
|
+
return typeval[nb] if typeval[nb]
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
# Validates taktuk options
|
199
|
+
# @api private
|
200
|
+
class Options < Hash
|
201
|
+
TAKTUK_VALID = [
|
202
|
+
'begin-group', 'connector', 'dynamic', 'end-group', 'machines-file',
|
203
|
+
'login', 'machine', 'self-propagate', 'dont-self-propagate',
|
204
|
+
'args-file', 'gateway', 'perl-interpreter', 'localhost',
|
205
|
+
'send-files', 'taktuk-command', 'path-value', 'command-separator',
|
206
|
+
'escape-character', 'option-separator', 'output-redirect',
|
207
|
+
'worksteal-behavior', 'time-granularity', 'no-numbering', 'timeout',
|
208
|
+
'cache-limit', 'window','window-adaptation','not-root','debug'
|
209
|
+
]
|
210
|
+
WRAPPER_VALID = [ 'streams', 'port', 'keys', 'user', 'config' ] # user is an alias for login
|
211
|
+
|
212
|
+
def check(optname)
|
213
|
+
ret = optname.to_s.gsub(/_/,'-').strip
|
214
|
+
raise ArgumentError.new("Invalid TakTuk option '--#{ret}'") unless TAKTUK_VALID.include?(ret)
|
215
|
+
ret
|
216
|
+
end
|
217
|
+
|
218
|
+
def to_cmd
|
219
|
+
|
220
|
+
self[:login] = self[:user] if keys.include?(:user)
|
221
|
+
self.keys.inject([]) do |ret,opt|
|
222
|
+
if not WRAPPER_VALID.include?(opt.to_s) then
|
223
|
+
ret << "--#{check(opt)}"
|
224
|
+
if self[opt]
|
225
|
+
if self[opt].is_a?(String)
|
226
|
+
ret << self[opt] unless self[opt].empty?
|
227
|
+
else
|
228
|
+
ret << self[opt].to_s
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
ret
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
# Generates a taktuk CLI compatible host list
|
238
|
+
# @api private
|
239
|
+
class Hostlist
|
240
|
+
def initialize(hostlist)
|
241
|
+
@hostlist=hostlist
|
242
|
+
end
|
243
|
+
|
244
|
+
def free
|
245
|
+
@hostlist = nil
|
246
|
+
end
|
247
|
+
|
248
|
+
def exclude(node)
|
249
|
+
@hostlist.remove(node) if @hostlist.is_a?(Array)
|
250
|
+
end
|
251
|
+
|
252
|
+
def to_cmd
|
253
|
+
ret = []
|
254
|
+
if @hostlist.is_a?(Array)
|
255
|
+
@hostlist.each do |host|
|
256
|
+
ret << '-m'
|
257
|
+
ret << host
|
258
|
+
end
|
259
|
+
elsif @hostlist.is_a?(String)
|
260
|
+
ret << '-f'
|
261
|
+
ret << @hostlist
|
262
|
+
end
|
263
|
+
ret
|
264
|
+
end
|
265
|
+
|
266
|
+
def to_a
|
267
|
+
if @hostlist.is_a?(Array)
|
268
|
+
@hostlist
|
269
|
+
elsif @hostlist.is_a?(String)
|
270
|
+
raise "Hostfile does not exist" unless File.exist?(@hostlist)
|
271
|
+
File.read(@hostlist).split("\n").uniq
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
276
|
+
# Validates the commands accepted by taktuk
|
277
|
+
# @api private
|
278
|
+
class Commands < Array
|
279
|
+
TOKENS=[
|
280
|
+
'broadcast', 'downcast', 'exec', 'get', 'put', 'input', 'data',
|
281
|
+
'file', 'pipe', 'close', 'line', 'target', 'kill', 'message',
|
282
|
+
'network', 'state', 'cancel', 'renumber', 'update', 'option',
|
283
|
+
'synchronize', 'taktuk_perl', 'quit', 'wait', 'reduce'
|
284
|
+
]
|
285
|
+
|
286
|
+
def <<(val)
|
287
|
+
raise ArgumentError.new("'Invalid TakTuk command '#{val}'") unless check(val)
|
288
|
+
super(val)
|
289
|
+
end
|
290
|
+
|
291
|
+
def check(val)
|
292
|
+
if val =~ /^-?\[.*-?\]$|^;$/
|
293
|
+
true
|
294
|
+
elsif val.nil? or val.empty?
|
295
|
+
false
|
296
|
+
else
|
297
|
+
tmp = val.split(' ',2)
|
298
|
+
return false unless valid?(tmp[0])
|
299
|
+
if !tmp[1].nil? and !tmp[1].empty?
|
300
|
+
check(tmp[1])
|
301
|
+
else
|
302
|
+
true
|
303
|
+
end
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
def valid?(value)
|
308
|
+
TOKENS.each do |token|
|
309
|
+
return true if token =~ /^#{Regexp.escape(value)}.*$/
|
310
|
+
end
|
311
|
+
return false
|
312
|
+
end
|
313
|
+
|
314
|
+
def to_cmd
|
315
|
+
self.inject([]) do |ret,val|
|
316
|
+
if val =~ /^\[(.*)\]$/
|
317
|
+
ret += ['[',Regexp.last_match(1).strip,']']
|
318
|
+
else
|
319
|
+
ret += val.split(' ')
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
# This class wraps the command TakTuk and generates automatically
|
326
|
+
# the long CLI options for taktuk command.
|
327
|
+
class TakTuk
|
328
|
+
attr_accessor :streams,:binary
|
329
|
+
attr_reader :stdout,:stderr,:status, :args, :exec_cmd, :commands
|
330
|
+
|
331
|
+
VALID_STREAMS = [:output, :error, :status, :connector, :state, :info, :message, :taktuk ]
|
332
|
+
|
333
|
+
def initialize(hostlist,options = {:connector => 'ssh'})
|
334
|
+
raise ArgumentError.new("options parameter has to be a hash") unless options.is_a?(Hash)
|
335
|
+
|
336
|
+
@binary = 'taktuk'
|
337
|
+
@options = Options[options.merge({ :streams => [:output, :error, :status ]})] if options[:streams].nil?
|
338
|
+
@options.merge!({:connector => 'ssh'}) if options[:connector].nil?
|
339
|
+
@streams = Stream.new(@options[:streams])
|
340
|
+
# @streams = Stream.new([:output,:error,:status, :state])
|
341
|
+
|
342
|
+
@hostlist = Hostlist.new(hostlist)
|
343
|
+
@commands = Commands.new
|
344
|
+
|
345
|
+
@args = nil
|
346
|
+
@stdout = nil
|
347
|
+
@stderr = nil
|
348
|
+
@status = nil
|
349
|
+
|
350
|
+
@exec_cmd = nil
|
351
|
+
@curthread = nil
|
352
|
+
@connector = @options[:connector]
|
353
|
+
end
|
354
|
+
|
355
|
+
def run!(opts = {})
|
356
|
+
@curthread = Thread.current
|
357
|
+
@args = []
|
358
|
+
@args += @options.to_cmd
|
359
|
+
|
360
|
+
@streams.types.each{ |name|
|
361
|
+
@args << '-o'
|
362
|
+
@args << "#{name.to_s}=#{@streams.to_cmd}"
|
363
|
+
}
|
364
|
+
|
365
|
+
connector = build_connector
|
366
|
+
@args += ["--connector", "#{connector}"] unless connector.nil?
|
367
|
+
|
368
|
+
@args += @hostlist.to_cmd
|
369
|
+
@args += @commands.to_cmd
|
370
|
+
|
371
|
+
hosts = @hostlist.to_a
|
372
|
+
outputs_size = opts[:outputs_size] || 0
|
373
|
+
@exec_cmd = Cute::Execute[@binary,*@args].run!(
|
374
|
+
:stdout_size => outputs_size * hosts.size,
|
375
|
+
:stderr_size => outputs_size * hosts.size,
|
376
|
+
:stdin => false
|
377
|
+
)
|
378
|
+
@status, @stdout, @stderr, emptypipes = @exec_cmd.wait({:checkstatus=>false})
|
379
|
+
|
380
|
+
unless @status.success?
|
381
|
+
@curthread = nil
|
382
|
+
return false
|
383
|
+
end
|
384
|
+
|
385
|
+
unless emptypipes
|
386
|
+
@curthread = nil
|
387
|
+
@stderr = "Too much data on the TakTuk command's stdout/stderr"
|
388
|
+
return false
|
389
|
+
end
|
390
|
+
|
391
|
+
results = @streams.parse(@stdout)
|
392
|
+
@curthread = nil
|
393
|
+
|
394
|
+
results
|
395
|
+
end
|
396
|
+
|
397
|
+
# It executes the commands so far stored in the @commands variable
|
398
|
+
# and reinitialize the variable for post utilization.
|
399
|
+
def loop ()
|
400
|
+
run!()
|
401
|
+
$stdout.print(@stdout)
|
402
|
+
$stderr.print(@stderr)
|
403
|
+
@commands = Commands.new
|
404
|
+
end
|
405
|
+
|
406
|
+
def kill!()
|
407
|
+
unless @exec.nil?
|
408
|
+
@exec.kill
|
409
|
+
@exec = nil
|
410
|
+
end
|
411
|
+
free!()
|
412
|
+
end
|
413
|
+
|
414
|
+
def free!()
|
415
|
+
@binary = nil
|
416
|
+
@options = nil
|
417
|
+
# if @streams
|
418
|
+
# @streams.each_value do |stream|
|
419
|
+
# stream.free if stream
|
420
|
+
# stream = nil
|
421
|
+
# end
|
422
|
+
# end
|
423
|
+
@hostlist.free if @hostlist
|
424
|
+
@hostlist = nil
|
425
|
+
@commands = nil
|
426
|
+
@args = nil
|
427
|
+
@stdout = nil
|
428
|
+
@stderr = nil
|
429
|
+
@status = nil
|
430
|
+
@exec = nil
|
431
|
+
@curthread = nil
|
432
|
+
end
|
433
|
+
|
434
|
+
def raw!(string)
|
435
|
+
@commands << string.strip
|
436
|
+
self
|
437
|
+
end
|
438
|
+
|
439
|
+
# It executes a command on multiple hosts.
|
440
|
+
# All output is printed via *stdout* and *stderr*.
|
441
|
+
# Note that this method returns immediately,
|
442
|
+
# and requires a call to the loop method in order
|
443
|
+
# for the command to actually execute.
|
444
|
+
# The execution is done by TakTuk using broadcast exec.
|
445
|
+
# @param [String] cmd Command to execute.
|
446
|
+
#
|
447
|
+
# = Example
|
448
|
+
#
|
449
|
+
# tak.exec("hostname")
|
450
|
+
# tak.exec("mkdir ~/test")
|
451
|
+
# tak.loop() # to trigger the execution of commands
|
452
|
+
def exec(cmd)
|
453
|
+
mode = "broadcast"
|
454
|
+
@commands << "#{mode} exec"
|
455
|
+
@commands << "[ #{cmd} ]"
|
456
|
+
@commands << ';' # TakTuk command separator
|
457
|
+
end
|
458
|
+
|
459
|
+
# It transfers a file to all the machines in parallel.
|
460
|
+
# @param [String] source Source path for the file to be transfer
|
461
|
+
# @param [String] dest Destination path for the file to be transfer
|
462
|
+
#
|
463
|
+
# = Example
|
464
|
+
#
|
465
|
+
# tak.put("hosts.allow_template", "/etc/hosts.allow")
|
466
|
+
#
|
467
|
+
def put(source,dest)
|
468
|
+
mode = "broadcast"
|
469
|
+
@commands << "#{mode} put"
|
470
|
+
@commands << "[ #{source} ]"
|
471
|
+
@commands << "[ #{dest} ]"
|
472
|
+
@commands << ';' # TakTuk command separator
|
473
|
+
end
|
474
|
+
|
475
|
+
|
476
|
+
# It executes a command on multiple hosts capturing the output,
|
477
|
+
# and other information related with the execution.
|
478
|
+
# It blocks until the command finishes.
|
479
|
+
# @param [String] cmd Command to be executed
|
480
|
+
# @return [Hash] Result data structure
|
481
|
+
#
|
482
|
+
# = Example
|
483
|
+
#
|
484
|
+
# tak.exec!("uname -r") #=> {"node2"=>{:output=>"3.2.0-4-amd64", :status=>0}, "node3"=>{:output=>"3.2.0-4-amd64", :status=>0}, ...}
|
485
|
+
#
|
486
|
+
def exec!(cmd)
|
487
|
+
loop() unless @commands.empty?
|
488
|
+
exec(cmd)
|
489
|
+
results = run!()
|
490
|
+
@commands = Commands.new
|
491
|
+
return results
|
492
|
+
end
|
493
|
+
|
494
|
+
# Manages the taktuk command input
|
495
|
+
# @param [Hash] opts Options for the type of data
|
496
|
+
# @option opts [String] :data Raw data to be used as the input of a command
|
497
|
+
# @option opts [String] :filename a file to be used as the input of a command
|
498
|
+
#
|
499
|
+
# = Example
|
500
|
+
#
|
501
|
+
# tak.exec("wc -w")
|
502
|
+
# tak.input(:data => "data data data data")
|
503
|
+
#
|
504
|
+
# tak.exec("tar xvf -")
|
505
|
+
# tak.input(:file => "test_file.tar")
|
506
|
+
def input(opts = {})
|
507
|
+
mode = "broadcast"
|
508
|
+
@commands << "#{mode} input #{opts.keys.first.to_s}"
|
509
|
+
@commands << "[ #{opts.values.first} ]"
|
510
|
+
@commands << ';'
|
511
|
+
end
|
512
|
+
|
513
|
+
|
514
|
+
def [](command,prefix='[',suffix=']')
|
515
|
+
@commands << "#{prefix} #{command} #{suffix}"
|
516
|
+
self
|
517
|
+
end
|
518
|
+
|
519
|
+
def method_missing(meth,*args)
|
520
|
+
@commands << (meth.to_s.gsub(/_/,' ').strip.downcase)
|
521
|
+
args.each do |arg|
|
522
|
+
@commands.push(arg.strip.downcase)
|
523
|
+
end
|
524
|
+
self
|
525
|
+
end
|
526
|
+
|
527
|
+
alias close free!
|
528
|
+
|
529
|
+
private
|
530
|
+
# It builds a custom connector for TakTuk
|
531
|
+
def build_connector()
|
532
|
+
ssh_options = [:keys, :port, :config]
|
533
|
+
connector = @connector
|
534
|
+
if @options.keys.map{ |opt| ssh_options.include?(opt)}.any?
|
535
|
+
connector += " -p #{@options[:port]}" if @options[:port]
|
536
|
+
if @options[:keys]
|
537
|
+
keys = @options[:keys].is_a?(Array) ? @options[:keys].first : @options[:keys]
|
538
|
+
connector += " -i #{keys}"
|
539
|
+
end
|
540
|
+
connector += " -F #{@options[:config]}" if @options[:config]
|
541
|
+
end
|
542
|
+
return connector
|
543
|
+
end
|
544
|
+
|
545
|
+
# It is used to separate the commands, they will run in parallel.
|
546
|
+
def seq!
|
547
|
+
@commands << ';'
|
548
|
+
self
|
549
|
+
end
|
550
|
+
|
551
|
+
end
|
552
|
+
|
553
|
+
end
|
554
|
+
end
|