weave 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ /.vagrant
4
+ /.yardoc/
5
+ /doc/
@@ -0,0 +1 @@
1
+ --no-private --protected --markup=markdown -- lib/**/*.rb
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in weave.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Caleb Spare
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,28 @@
1
+ # Weave
2
+
3
+ Simple parallel ssh.
4
+
5
+ ## Implemented features
6
+
7
+ * Connection caching
8
+ * Parallel execution (with thread/connection limit)
9
+ * Serial execution
10
+ * Commands:
11
+
12
+ - `run`
13
+
14
+ ## Other ideas
15
+
16
+ * Commands:
17
+
18
+ - `sudo`
19
+ - `rsync`
20
+ - `put`?
21
+ - `get`?
22
+
23
+ * Handle password prompts (maybe just sudo prompt)
24
+ * Handle ctrl-c correctly?
25
+
26
+ ## To do:
27
+
28
+ * Full readme with detailed usage
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+ require "rake/testtask"
4
+
5
+ task :test => ["test:integrations"]
6
+
7
+ namespace :test do
8
+ Rake::TestTask.new(:integrations) do |task|
9
+ task.test_files = FileList["test/integrations/**/*.rb"]
10
+ end
11
+ end
@@ -0,0 +1,21 @@
1
+ Vagrant::Config.run do |config|
2
+ def apply_common_config_options(c)
3
+ c.vm.box = "lucid64"
4
+ c.vm.provision :shell, :inline =>
5
+ "sudo mkdir -p /root/.ssh && sudo cp /home/vagrant/.ssh/authorized_keys /root/.ssh/"
6
+ end
7
+
8
+ config.vm.define :weave1 do |config|
9
+ config.vm.host_name = "weave1"
10
+ config.vm.network :hostonly, "1.2.3.90"
11
+ config.vm.forward_port 22, 3220 # ssh
12
+ apply_common_config_options(config)
13
+ end
14
+
15
+ config.vm.define :weave2 do |config|
16
+ config.vm.host_name = "weave2"
17
+ config.vm.network :hostonly, "1.2.3.91"
18
+ config.vm.forward_port 22, 3221 #ssh
19
+ apply_common_config_options(config)
20
+ end
21
+ end
@@ -0,0 +1,7 @@
1
+ $:.unshift File.join(File.dirname(__FILE__), "../lib")
2
+
3
+ require "weave"
4
+
5
+ pool = Weave.connect(ARGV) do
6
+ run "ls"
7
+ end
@@ -0,0 +1,26 @@
1
+ $:.unshift File.join(File.dirname(__FILE__), "../lib")
2
+
3
+ require "readline"
4
+ require "weave"
5
+
6
+ usage = <<-EOS
7
+ Usage:
8
+ $ ./parallel-ssh user@host1 user@host2 ...
9
+ EOS
10
+
11
+ abort usage if ARGV.empty?
12
+
13
+ $stty_state = `stty -g`.chomp
14
+ $pool = Weave.connect(ARGV)
15
+
16
+ while command = Readline.readline(">>> ", true)
17
+ break unless command # ctrl-D
18
+ command.chomp!
19
+ next if command.empty?
20
+ break if ["exit", "quit"].include? command
21
+ $pool.execute { run command }
22
+ end
23
+
24
+ $pool.disconnect!
25
+ `stty #{$stty_state}`
26
+ puts "Bye."
@@ -0,0 +1,171 @@
1
+ require "net/ssh"
2
+ require "thread"
3
+ require "singleton"
4
+
5
+ module Weave
6
+ DEFAULT_THREAD_POOL_SIZE = 10
7
+
8
+ # @private
9
+ COLORS = {
10
+ red: 1,
11
+ green: 2,
12
+ yellow: 3,
13
+ blue: 4,
14
+ magenta: 5,
15
+ cyan: 6,
16
+ white: 7,
17
+ default: 8
18
+ }
19
+
20
+ # Create a connection pool for an array of hosts. Each host must have a user specified (e.g.,
21
+ # root@example.com). If a block is given, then the options are passed through to the underlying
22
+ # ConnectionPool and the block is immediately run in the context of each connection. Otherwise, a pool is
23
+ # returned.
24
+ #
25
+ # @see ConnectionPool#execute
26
+ def self.connect(host_list, options = {}, &block)
27
+ pool = ConnectionPool.new(host_list)
28
+ if block_given?
29
+ pool.execute(options, &block)
30
+ pool.disconnect!
31
+ else
32
+ return pool
33
+ end
34
+ end
35
+
36
+ # @private
37
+ def self.color_string(string, color) "\e[01;#{COLORS[color]+30}m#{string}\e[m" end
38
+
39
+ # Spread work, identified by a key, across multiple threads.
40
+ # @private
41
+ def self.with_thread_pool(keys, thread_pool_size, &block)
42
+ work_queue = Queue.new
43
+ mutex = Mutex.new
44
+ keys.each { |key| work_queue << key }
45
+
46
+ threads = (1..thread_pool_size).map do |i|
47
+ Thread.new do
48
+ begin
49
+ while (key = work_queue.pop(true))
50
+ yield key, mutex
51
+ end
52
+ rescue ThreadError # Queue is empty
53
+ end
54
+ end
55
+ end
56
+ threads.each(&:join)
57
+ nil
58
+ end
59
+
60
+ # A pool of SSH connections. Operations over the pool may be performed in serial or in parallel.
61
+ class ConnectionPool
62
+ # @param [Array] host_list the array of hosts, of the form user@host
63
+ def initialize(host_list)
64
+ @connections = host_list.reduce({}) { |pool, host| pool.merge(host => LazyConnection.new(host)) }
65
+ end
66
+
67
+ # Run a command over the connection pool. The block is evaluated in the context of LazyConnection.
68
+ #
69
+ # @param [Hash] options the various knobs
70
+ # @option options [Fixnum] :num_threads the number of concurrent threads to use to process this command.
71
+ # Defaults to `DEFAULT_THREAD_POOL_SIZE`.
72
+ # @option options [Boolean] :serial whether to process the command for each connection one at a time.
73
+ # @option options [Fixnum] :batch_by if set, group the connections into batches of no more than this value
74
+ # and fully process each batch before starting the next one.
75
+ def execute(options = {}, &block)
76
+ options[:num_threads] ||= DEFAULT_THREAD_POOL_SIZE
77
+ if options[:serial]
78
+ @connections.each_key { |host| @connections[host].self_eval &block }
79
+ elsif options[:batch_by]
80
+ @connections.each_key.each_slice(options[:batch_by]) do |batch|
81
+ Weave.with_thread_pool(batch, options[:num_threads]) do |host, mutex|
82
+ @connections[host].self_eval mutex, &block
83
+ end
84
+ end
85
+ else
86
+ Weave.with_thread_pool(@connections.keys, options[:num_threads]) do |host, mutex|
87
+ @connections[host].self_eval mutex, &block
88
+ end
89
+ end
90
+ end
91
+
92
+ # Disconnect all open connections.
93
+ def disconnect!() @connections.each_value(&:disconnect) end
94
+ end
95
+
96
+ # @private
97
+ class NilMutex
98
+ include Singleton
99
+ def synchronize() yield end
100
+ end
101
+
102
+ # An SSH connection which isn't established until it's needed.
103
+ class LazyConnection
104
+ # The username for the ssh connection
105
+ attr_reader :user
106
+ # The hostname for the ssh connection
107
+ attr_reader :host
108
+
109
+ # @param [String] host_string the host of the form user@host
110
+ def initialize(host_string)
111
+ @user, @host = self.class.user_and_host(host_string)
112
+ @connection = nil
113
+ @mutex = NilMutex.instance
114
+ end
115
+
116
+ # Run a command on this connection. This will open a connection if it's not already connected. The way the
117
+ # output is presented is determined by the option `:output`. The default, `:output => :pretty`, prints
118
+ # each line of output with the name of the host and whether the output is stderr or stdout. If `:output =>
119
+ # :raw`, then the output will be passed as is directly back to `STDERR` or `STDOUT` as appropriate. If
120
+ # `:output => :capture`, then this method returns the output in a hash of the form
121
+ # `{ :stdout => [...], :stderr => [...] }`.
122
+ #
123
+ # @param [Hash] options the output options
124
+ # @option options [Symbol] :output the output format
125
+ def run(command, options = {})
126
+ options[:output] ||= :pretty
127
+ @connection ||= Net::SSH.start(@host, @user)
128
+ output = { :stderr => [], :stdout => [] }
129
+ @connection.exec(command) do |channel, stream, data|
130
+ case options[:output]
131
+ when :capture
132
+ output[stream] << data
133
+ when :raw
134
+ out_stream = stream == :stdout ? STDOUT : STDERR
135
+ out_stream.print data
136
+ else
137
+ stream_colored = case stream
138
+ when :stdout then Weave.color_string("out", :green)
139
+ when :stderr then Weave.color_string("err", :red)
140
+ end
141
+ lines = data.split("\n").map { |line| "[#{stream_colored}|#{host}] #{line}" }.join("\n")
142
+ @mutex.synchronize { puts lines }
143
+ end
144
+ end
145
+ @connection.loop(0.1)
146
+ (options[:output] == :capture) ? output : nil
147
+ end
148
+
149
+ # @private
150
+ def self.user_and_host(host_string)
151
+ user, at, host = host_string.rpartition("@")
152
+ if [user, host].any? { |part| part.nil? || part.empty? }
153
+ raise "Bad hostname (needs to be of the form user@host): #{host_string}"
154
+ end
155
+ [user, host]
156
+ end
157
+
158
+ # @private
159
+ def self_eval(mutex = nil, &block)
160
+ @mutex = mutex || NilMutex.instance
161
+ instance_eval &block
162
+ @mutex = NilMutex.instance
163
+ end
164
+
165
+ # Disconnect, if connected.
166
+ def disconnect()
167
+ @connection.close if @connection
168
+ @connection = nil
169
+ end
170
+ end
171
+ end
@@ -0,0 +1,3 @@
1
+ module Weave
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,75 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), "../test_helper"))
2
+
3
+ require "net/ssh"
4
+ require "weave"
5
+
6
+ class SanityTest < Scope::TestCase
7
+ TEST_HOSTS = [1, 2].map { |i| "weave#{i}" }
8
+ ROOT_AT_TEST_HOSTS = TEST_HOSTS.map { |host| "root@#{host}" }
9
+
10
+ setup_once do
11
+ # Make sure the machines are up.
12
+ vagrant_status = `bundle exec vagrant status`
13
+ unless $?.to_i.zero? && TEST_HOSTS.each { |host| vagrant_status =~ /#{host}\w+running/ }
14
+ abort "You need to set up the test vagrant virtual machines to run the sanity test." \
15
+ "Run 'bundle exec vagrant up'."
16
+ end
17
+ # Make sure the user's ssh config has weave entries.
18
+ TEST_HOSTS.each do |host|
19
+ config = Net::SSH::Config.load("~/.ssh/config", host)
20
+ unless config["hostname"] == "127.0.0.1"
21
+ abort "You need to add weave{1,2} to your ~/.ssh/config." \
22
+ "You can use the output of 'bundle exec vagrant ssh-config weave1'"
23
+ end
24
+ end
25
+ end
26
+
27
+ context "executing some commands" do
28
+ should "run some simple commands" do
29
+ output = Hash.new { |h, k| h[k] = [] }
30
+ Weave.connect(ROOT_AT_TEST_HOSTS) do
31
+ output[host] = run("echo 'hello'", :output => :capture)
32
+ end
33
+ TEST_HOSTS.each do |host|
34
+ assert_empty output[host][:stderr]
35
+ assert_equal ["hello\n"], output[host][:stdout]
36
+ end
37
+ end
38
+
39
+ context "in serial" do
40
+ should "run some commands in the expected order" do
41
+ output = []
42
+ Weave.connect(ROOT_AT_TEST_HOSTS, :serial => true) do
43
+ command = (host == "weave1") ? "sleep 0.2; echo 'delayed'" : "echo 'on time'"
44
+ output += run(command, :output => :capture)[:stdout]
45
+ end
46
+ assert_equal ["delayed\n", "on time\n"], output
47
+ end
48
+ end
49
+
50
+ context "in parallel" do
51
+ should "run some commands in the expected order" do
52
+ output = []
53
+ Weave.connect(ROOT_AT_TEST_HOSTS) do
54
+ command = (host == "weave1") ? "sleep 0.2; echo 'delayed'" : "echo 'on time'"
55
+ result = run(command, :output => :capture)
56
+ output += result[:stdout]
57
+ end
58
+ assert_equal ["on time\n", "delayed\n"], output
59
+ end
60
+ end
61
+
62
+ context "on a connection pool" do
63
+ should "run basic commands" do
64
+ output = Hash.new { |h, k| h[k] = [] }
65
+ Weave.connect(ROOT_AT_TEST_HOSTS).execute do
66
+ output[host] = run("echo 'hello'", :output => :capture)
67
+ end
68
+ TEST_HOSTS.each do |host|
69
+ assert_empty output[host][:stderr]
70
+ assert_equal ["hello\n"], output[host][:stdout]
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,4 @@
1
+ $:.unshift File.join(File.dirname(__FILE__), "../lib")
2
+
3
+ require "scope"
4
+ require "minitest/autorun"
@@ -0,0 +1,25 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/weave/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.authors = ["Caleb Spare"]
6
+ gem.email = ["cespare@gmail.com"]
7
+ gem.description = %q{Simple parallel ssh.}
8
+ gem.summary = %q{Simple parallel ssh.}
9
+ gem.homepage = ""
10
+
11
+ gem.files = `git ls-files`.split($\)
12
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
+ gem.name = "weave"
15
+ gem.require_paths = ["lib"]
16
+ gem.version = Weave::VERSION
17
+
18
+ # For running integration tests.
19
+ gem.add_development_dependency "vagrant"
20
+ gem.add_development_dependency "scope"
21
+ gem.add_development_dependency "rake"
22
+ # For generating the docs
23
+ gem.add_development_dependency "yard"
24
+ gem.add_development_dependency "redcarpet"
25
+ end
metadata ADDED
@@ -0,0 +1,142 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: weave
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Caleb Spare
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-10-12 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: vagrant
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: scope
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ - !ruby/object:Gem::Dependency
47
+ name: rake
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ - !ruby/object:Gem::Dependency
63
+ name: yard
64
+ requirement: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ type: :development
71
+ prerelease: false
72
+ version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ - !ruby/object:Gem::Dependency
79
+ name: redcarpet
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ! '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ type: :development
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ description: Simple parallel ssh.
95
+ email:
96
+ - cespare@gmail.com
97
+ executables: []
98
+ extensions: []
99
+ extra_rdoc_files: []
100
+ files:
101
+ - .gitignore
102
+ - .yardopts
103
+ - Gemfile
104
+ - LICENSE
105
+ - README.md
106
+ - Rakefile
107
+ - Vagrantfile
108
+ - examples/basic.rb
109
+ - examples/parallel-console.rb
110
+ - lib/weave.rb
111
+ - lib/weave/version.rb
112
+ - test/integrations/sanity_test.rb
113
+ - test/test_helper.rb
114
+ - weave.gemspec
115
+ homepage: ''
116
+ licenses: []
117
+ post_install_message:
118
+ rdoc_options: []
119
+ require_paths:
120
+ - lib
121
+ required_ruby_version: !ruby/object:Gem::Requirement
122
+ none: false
123
+ requirements:
124
+ - - ! '>='
125
+ - !ruby/object:Gem::Version
126
+ version: '0'
127
+ required_rubygems_version: !ruby/object:Gem::Requirement
128
+ none: false
129
+ requirements:
130
+ - - ! '>='
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ requirements: []
134
+ rubyforge_project:
135
+ rubygems_version: 1.8.23
136
+ signing_key:
137
+ specification_version: 3
138
+ summary: Simple parallel ssh.
139
+ test_files:
140
+ - test/integrations/sanity_test.rb
141
+ - test/test_helper.rb
142
+ has_rdoc: