capissh 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +18 -0
- data/README.md +37 -0
- data/lib/capissh.rb +43 -0
- data/lib/capissh/command.rb +197 -0
- data/lib/capissh/command/tree.rb +138 -0
- data/lib/capissh/configuration.rb +65 -0
- data/lib/capissh/connection_manager.rb +250 -0
- data/lib/capissh/errors.rb +19 -0
- data/lib/capissh/file_transfers.rb +54 -0
- data/lib/capissh/invocation.rb +278 -0
- data/lib/capissh/logger.rb +157 -0
- data/lib/capissh/processable.rb +54 -0
- data/lib/capissh/server_definition.rb +111 -0
- data/lib/capissh/ssh.rb +110 -0
- data/lib/capissh/transfer.rb +218 -0
- data/lib/capissh/version.rb +3 -0
- metadata +204 -0
data/LICENSE
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
2
|
+
a copy of this software and associated documentation files (the
|
3
|
+
"Software"), to deal in the Software without restriction, including
|
4
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
5
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
6
|
+
permit persons to whom the Software is furnished to do so, subject to
|
7
|
+
the following conditions:
|
8
|
+
|
9
|
+
The above copyright notice and this permission notice shall be
|
10
|
+
included in all copies or substantial portions of the Software.
|
11
|
+
|
12
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
13
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
14
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
15
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
16
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
17
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
18
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# Capissh?
|
2
|
+
|
3
|
+
An extraction of Capistrano's parallel SSH command execution, capiche?
|
4
|
+
|
5
|
+
![Don Vito Corleone](http://i.imgur.com/hAcWI.jpg)
|
6
|
+
|
7
|
+
|
8
|
+
## About
|
9
|
+
|
10
|
+
Capissh executes commands (and soon transfers) on remote servers in parallel.
|
11
|
+
|
12
|
+
Capissh will maintain open connections with servers it has seen when is is
|
13
|
+
sensible to do so. When the batch size is not restricted, Capissh will maintain
|
14
|
+
all connections, which greatly reduces connection overhead. Sets of commands
|
15
|
+
are run in parallel on all servers within the batch.
|
16
|
+
|
17
|
+
## Example
|
18
|
+
|
19
|
+
The interface is intentionally simple. A Capissh::Configuration object will
|
20
|
+
maintain the open sessions, which will be matched up with servers for each
|
21
|
+
command invocation.
|
22
|
+
|
23
|
+
require 'capissh'
|
24
|
+
|
25
|
+
servers = ['user@host.com', 'user@example.com']
|
26
|
+
capissh = Capissh.new
|
27
|
+
capissh.run servers, 'date'
|
28
|
+
capissh.run servers, 'uname -a'
|
29
|
+
capissh.sudo servers, 'ls /root'
|
30
|
+
|
31
|
+
## Thank You
|
32
|
+
|
33
|
+
Huge thank you to Jamis Buck and all the other Capistrano contributors for
|
34
|
+
creating and maintaining this gem. Without them, this library would not exist,
|
35
|
+
and ruby deployment wouldn't be at the level it is today.
|
36
|
+
|
37
|
+
Most of this code is directly extracted from capistrano without modification.
|
data/lib/capissh.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# NOTES
|
2
|
+
#
|
3
|
+
# Parts
|
4
|
+
#
|
5
|
+
# * Server definition - Some object that can be executed on to connect to a server
|
6
|
+
# * Server collection - A collection of servers on which to execute
|
7
|
+
# * Command - What to run on the servers
|
8
|
+
# * Command Tree - What to run in parallel
|
9
|
+
#
|
10
|
+
# SSH Related stuff
|
11
|
+
# * Connection - A connection to a server definition that is held for reuse
|
12
|
+
# * Connection pool - All the connections so persistent connections can be found
|
13
|
+
# * Connection factory - Creates a persistent connection from a server definition
|
14
|
+
#
|
15
|
+
# Set of servers
|
16
|
+
# Command
|
17
|
+
# Run this command on this set of servers...
|
18
|
+
# Using this connector (or the default ssh connector)
|
19
|
+
#
|
20
|
+
# State to be kept somewhere:
|
21
|
+
# servers with sessions
|
22
|
+
# * set of sessions (if sessions are being persisted, with max hosts set, this could be a subset that gets blown away)
|
23
|
+
# * set of servers, connected to sessions (this needs to be filterable by the user)
|
24
|
+
# these are both 1 to 1
|
25
|
+
# need to sort, constrain, etc
|
26
|
+
# * Capissh init options
|
27
|
+
#
|
28
|
+
#
|
29
|
+
# connector receives a server (set of servers?) and a command (command tree?)
|
30
|
+
# functionality: to interogate servers and figure out which command should
|
31
|
+
# run on each server and then hand out the command to the appropriate server.
|
32
|
+
# connector steps through servers (or command tree) and yields the command to each server
|
33
|
+
# Yielding expects each server to run the command given (or maybe it just has the
|
34
|
+
# chance to modify the command?)
|
35
|
+
#
|
36
|
+
|
37
|
+
require 'capissh/configuration'
|
38
|
+
|
39
|
+
module Capissh
|
40
|
+
def self.new(*args)
|
41
|
+
Capissh::Configuration.new(*args)
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,197 @@
|
|
1
|
+
require 'benchmark'
|
2
|
+
require 'capissh/errors'
|
3
|
+
require 'capissh/processable'
|
4
|
+
require 'capissh/command/tree'
|
5
|
+
|
6
|
+
module Capissh
|
7
|
+
|
8
|
+
# This class encapsulates a single command to be executed on a set of remote
|
9
|
+
# machines, in parallel.
|
10
|
+
class Command
|
11
|
+
include Processable
|
12
|
+
|
13
|
+
attr_reader :tree, :sessions, :options
|
14
|
+
|
15
|
+
class << self
|
16
|
+
attr_accessor :default_io_proc
|
17
|
+
|
18
|
+
def process_tree(tree, sessions, options={})
|
19
|
+
new(tree, sessions, options).process!
|
20
|
+
end
|
21
|
+
alias process process_tree
|
22
|
+
|
23
|
+
# Convenience method to process a command given as a string
|
24
|
+
# rather than a Command::Tree.
|
25
|
+
def process_string(string, sessions, options={}, &block)
|
26
|
+
tree = Tree.twig(nil, string, &block)
|
27
|
+
process_tree(tree, sessions, options)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
self.default_io_proc = Proc.new do |ch, stream, out|
|
32
|
+
if ch[:logger]
|
33
|
+
level = stream == :err ? :important : :info
|
34
|
+
ch[:logger].send(level, out, "#{stream} :: #{ch[:server]}")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# Instantiates a new command object. The +command+ must be a string
|
39
|
+
# containing the command to execute. +sessions+ is an array of Net::SSH
|
40
|
+
# session instances, and +options+ must be a hash containing any of the
|
41
|
+
# following keys:
|
42
|
+
#
|
43
|
+
# * +shell+: (optional), the shell (eg. 'bash') or false. Default: 'sh'
|
44
|
+
# * +logger+: (optional), a Capissh::Logger instance
|
45
|
+
# * +data+: (optional), a string to be sent to the command via it's stdin
|
46
|
+
# * +eof+: (optional), close stdin after sending data
|
47
|
+
# * +env+: (optional), a string or hash to be interpreted as environment
|
48
|
+
# variables that should be defined for this command invocation.
|
49
|
+
# * +pty+: (optional), execute the command in a pty
|
50
|
+
def initialize(tree, sessions, options={})
|
51
|
+
@tree = tree
|
52
|
+
@options = options
|
53
|
+
@sessions = sessions
|
54
|
+
@channels = open_channels
|
55
|
+
end
|
56
|
+
|
57
|
+
# Processes the command in parallel on all specified hosts. If the command
|
58
|
+
# fails (non-zero return code) on any of the hosts, this will raise a
|
59
|
+
# Capissh::CommandError.
|
60
|
+
def process!
|
61
|
+
elapsed = Benchmark.realtime do
|
62
|
+
loop do
|
63
|
+
break unless process_iteration { active? }
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
logger.trace "command finished in #{(elapsed * 1000).round}ms" if logger
|
68
|
+
|
69
|
+
failed = @channels.select { |ch| ch[:status] != 0 }
|
70
|
+
if failed.any?
|
71
|
+
commands = failed.inject({}) do |map, ch|
|
72
|
+
map[ch[:command]] ||= []
|
73
|
+
map[ch[:command]] << ch[:server]
|
74
|
+
map
|
75
|
+
end
|
76
|
+
message = commands.map { |command, list| "#{command.inspect} on #{list.join(',')}" }.join("; ")
|
77
|
+
error = CommandError.new("failed: #{message}")
|
78
|
+
error.hosts = commands.values.flatten
|
79
|
+
raise error
|
80
|
+
end
|
81
|
+
|
82
|
+
self
|
83
|
+
end
|
84
|
+
|
85
|
+
def active?
|
86
|
+
@channels.any? { |ch| !ch[:closed] }
|
87
|
+
end
|
88
|
+
|
89
|
+
# Force the command to stop processing, by closing all open channels
|
90
|
+
# associated with this command.
|
91
|
+
def stop!
|
92
|
+
@channels.each do |ch|
|
93
|
+
ch.close unless ch[:closed]
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
private
|
98
|
+
|
99
|
+
def logger
|
100
|
+
options[:logger]
|
101
|
+
end
|
102
|
+
|
103
|
+
def open_channels
|
104
|
+
sessions.map do |session|
|
105
|
+
server = session.xserver
|
106
|
+
@tree.base_command_and_callback(server).map do |base_command, io_proc|
|
107
|
+
session.open_channel do |channel|
|
108
|
+
channel[:server] = server
|
109
|
+
channel[:options] = options
|
110
|
+
channel[:logger] = logger
|
111
|
+
channel[:base_command] = base_command
|
112
|
+
channel[:io_proc] = io_proc
|
113
|
+
|
114
|
+
request_pty_if_necessary(channel) do |ch|
|
115
|
+
logger.trace "executing command", ch[:server] if logger
|
116
|
+
|
117
|
+
command_line = compose_command(channel[:base_command], channel[:server])
|
118
|
+
channel[:command] = command_line
|
119
|
+
|
120
|
+
ch.exec(command_line)
|
121
|
+
ch.send_data(options[:data]) if options[:data]
|
122
|
+
ch.eof! if options[:eof]
|
123
|
+
end
|
124
|
+
|
125
|
+
channel.on_data do |ch, data|
|
126
|
+
ch[:io_proc].call(ch, :out, data)
|
127
|
+
end
|
128
|
+
|
129
|
+
channel.on_extended_data do |ch, type, data|
|
130
|
+
ch[:io_proc].call(ch, :err, data)
|
131
|
+
end
|
132
|
+
|
133
|
+
channel.on_request("exit-status") do |ch, data|
|
134
|
+
ch[:status] = data.read_long
|
135
|
+
end
|
136
|
+
|
137
|
+
channel.on_close do |ch|
|
138
|
+
ch[:closed] = true
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end.flatten
|
143
|
+
end
|
144
|
+
|
145
|
+
def request_pty_if_necessary(channel)
|
146
|
+
if options[:pty]
|
147
|
+
channel.request_pty do |ch, success|
|
148
|
+
if success
|
149
|
+
yield ch
|
150
|
+
else
|
151
|
+
# just log it, don't actually raise an exception, since the
|
152
|
+
# process method will see that the status is not zero and will
|
153
|
+
# raise an exception then.
|
154
|
+
logger.important "could not open channel", ch[:server] if logger
|
155
|
+
ch.close
|
156
|
+
end
|
157
|
+
end
|
158
|
+
else
|
159
|
+
yield channel
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
def compose_command(command, server)
|
164
|
+
command = command.strip.gsub(/\r?\n/, "\\\n")
|
165
|
+
|
166
|
+
if options[:shell] == false
|
167
|
+
shell = nil
|
168
|
+
else
|
169
|
+
shell = "#{options[:shell] || "sh"} -c"
|
170
|
+
command = command.gsub(/'/) { |m| "'\\''" }
|
171
|
+
command = "'#{command}'"
|
172
|
+
end
|
173
|
+
|
174
|
+
[environment, shell, command].compact.join(" ")
|
175
|
+
end
|
176
|
+
|
177
|
+
# prepare a space-separated sequence of variables assignments
|
178
|
+
# intended to be prepended to a command, so the shell sets
|
179
|
+
# the environment before running the command.
|
180
|
+
# i.e.: options[:env] = {'PATH' => '/opt/ruby/bin:$PATH',
|
181
|
+
# 'TEST' => '( "quoted" )'}
|
182
|
+
# environment returns:
|
183
|
+
# "env TEST=(\ \"quoted\"\ ) PATH=/opt/ruby/bin:$PATH"
|
184
|
+
def environment
|
185
|
+
return if options[:env].nil? || options[:env].empty?
|
186
|
+
@environment ||=
|
187
|
+
if String === options[:env]
|
188
|
+
"env #{options[:env]}"
|
189
|
+
else
|
190
|
+
options[:env].inject("env") do |string, (name, value)|
|
191
|
+
value = value.to_s.gsub(/[ "]/) { |m| "\\#{m}" }
|
192
|
+
string << " #{name}=#{value}"
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
@@ -0,0 +1,138 @@
|
|
1
|
+
module Capissh
|
2
|
+
class Command
|
3
|
+
class Tree
|
4
|
+
attr_reader :configuration
|
5
|
+
attr_reader :branches
|
6
|
+
attr_reader :fallback
|
7
|
+
|
8
|
+
include Enumerable
|
9
|
+
|
10
|
+
class Branch
|
11
|
+
attr_accessor :command, :callback
|
12
|
+
attr_reader :options
|
13
|
+
|
14
|
+
def initialize(command, options, callback)
|
15
|
+
@command = command.strip.gsub(/\r?\n/, "\\\n")
|
16
|
+
@callback = callback || Capissh::Command.default_io_proc
|
17
|
+
@options = options
|
18
|
+
@skip = false
|
19
|
+
end
|
20
|
+
|
21
|
+
def last?
|
22
|
+
options[:last]
|
23
|
+
end
|
24
|
+
|
25
|
+
def skip?
|
26
|
+
@skip
|
27
|
+
end
|
28
|
+
|
29
|
+
def skip!
|
30
|
+
@skip = true
|
31
|
+
end
|
32
|
+
|
33
|
+
def match(server)
|
34
|
+
true
|
35
|
+
end
|
36
|
+
|
37
|
+
def to_s
|
38
|
+
command.inspect
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class ConditionBranch < Branch
|
43
|
+
attr_accessor :configuration
|
44
|
+
attr_accessor :condition
|
45
|
+
|
46
|
+
class Evaluator
|
47
|
+
attr_reader :configuration, :condition, :server
|
48
|
+
|
49
|
+
def initialize(config, condition, server)
|
50
|
+
@configuration = config
|
51
|
+
@condition = condition
|
52
|
+
@server = server
|
53
|
+
end
|
54
|
+
|
55
|
+
def in?(role)
|
56
|
+
configuration.roles[role].include?(server)
|
57
|
+
end
|
58
|
+
|
59
|
+
def result
|
60
|
+
eval(condition, binding)
|
61
|
+
end
|
62
|
+
|
63
|
+
def method_missing(sym, *args, &block)
|
64
|
+
if server.respond_to?(sym)
|
65
|
+
server.send(sym, *args, &block)
|
66
|
+
elsif configuration.respond_to?(sym)
|
67
|
+
configuration.send(sym, *args, &block)
|
68
|
+
else
|
69
|
+
super
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def initialize(configuration, condition, command, options, callback)
|
75
|
+
@configuration = configuration
|
76
|
+
@condition = condition
|
77
|
+
super(command, options, callback)
|
78
|
+
end
|
79
|
+
|
80
|
+
def match(server)
|
81
|
+
Evaluator.new(configuration, condition, server).result
|
82
|
+
end
|
83
|
+
|
84
|
+
def to_s
|
85
|
+
"#{condition.inspect} :: #{command.inspect}"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# A tree with only one branch.
|
90
|
+
def self.twig(config, command, &block)
|
91
|
+
new(config) { |t| t.else(command, &block) }
|
92
|
+
end
|
93
|
+
|
94
|
+
def initialize(config)
|
95
|
+
@configuration = config
|
96
|
+
@branches = []
|
97
|
+
yield self if block_given?
|
98
|
+
end
|
99
|
+
|
100
|
+
def when(condition, command, options={}, &block)
|
101
|
+
branches << ConditionBranch.new(configuration, condition, command, options, block)
|
102
|
+
end
|
103
|
+
|
104
|
+
def else(command, &block)
|
105
|
+
@fallback = Branch.new(command, {}, block)
|
106
|
+
end
|
107
|
+
|
108
|
+
def branches_for(server)
|
109
|
+
seen_last = false
|
110
|
+
matches = branches.select do |branch|
|
111
|
+
success = !seen_last && !branch.skip? && branch.match(server)
|
112
|
+
seen_last = success && branch.last?
|
113
|
+
success
|
114
|
+
end
|
115
|
+
|
116
|
+
matches << fallback if matches.empty? && fallback
|
117
|
+
|
118
|
+
return matches
|
119
|
+
end
|
120
|
+
|
121
|
+
def base_command_and_callback(server)
|
122
|
+
branches_for(server).map do |branch|
|
123
|
+
command = branch.command
|
124
|
+
if configuration
|
125
|
+
command = configuration.placeholder_callback.call(command, server)
|
126
|
+
end
|
127
|
+
[command, branch.callback]
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def each
|
132
|
+
branches.each { |branch| yield branch }
|
133
|
+
yield fallback if fallback
|
134
|
+
return self
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'capissh/logger'
|
2
|
+
require 'capissh/command'
|
3
|
+
require 'capissh/connection_manager'
|
4
|
+
require 'capissh/invocation'
|
5
|
+
require 'capissh/file_transfers'
|
6
|
+
require 'forwardable'
|
7
|
+
|
8
|
+
module Capissh
|
9
|
+
# Represents a specific Capissh configuration.
|
10
|
+
class Configuration
|
11
|
+
extend Forwardable
|
12
|
+
|
13
|
+
class << self
|
14
|
+
attr_accessor :default_placeholder_callback
|
15
|
+
end
|
16
|
+
|
17
|
+
self.default_placeholder_callback = proc do |command, server|
|
18
|
+
command.gsub(/\$CAPISSH:HOST\$/, server.host)
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_reader :logger, :options
|
22
|
+
|
23
|
+
def initialize(options={})
|
24
|
+
@options = options.dup
|
25
|
+
@logger = Capissh::Logger.new(@options)
|
26
|
+
@options[:default_environment] ||= {}
|
27
|
+
@options[:default_run_options] ||= {}
|
28
|
+
@options[:default_shell] ||= nil
|
29
|
+
end
|
30
|
+
|
31
|
+
def set(key, value)
|
32
|
+
@options[key.to_sym] = value
|
33
|
+
end
|
34
|
+
|
35
|
+
def fetch(key, *args, &block)
|
36
|
+
@options.fetch(key.to_sym, *args, &block)
|
37
|
+
end
|
38
|
+
|
39
|
+
def debug
|
40
|
+
fetch :debug, false
|
41
|
+
end
|
42
|
+
|
43
|
+
def dry_run
|
44
|
+
fetch :dry_run, false
|
45
|
+
end
|
46
|
+
|
47
|
+
def placeholder_callback
|
48
|
+
fetch :placeholder_callback, self.class.default_placeholder_callback
|
49
|
+
end
|
50
|
+
|
51
|
+
def connection_manager
|
52
|
+
@connection_manager ||= ConnectionManager.new(@options.merge(:logger => logger))
|
53
|
+
end
|
54
|
+
|
55
|
+
def file_transfers
|
56
|
+
@file_transfers ||= FileTransfers.new(self, connection_manager, :logger => logger)
|
57
|
+
end
|
58
|
+
def_delegators :file_transfers, :put, :get, :upload, :download, :transfer
|
59
|
+
|
60
|
+
def invocation
|
61
|
+
@invocation ||= Invocation.new(self, connection_manager, :logger => logger)
|
62
|
+
end
|
63
|
+
def_delegators :invocation, :parallel, :invoke_command, :run, :sudo, :sudo_command
|
64
|
+
end
|
65
|
+
end
|