rutty 2.3.2 → 2.4.0

data/Gemfile

@@ -2,13 +2,18 @@ source :gemcutter
 
 gem "commander", '~> 4.0.3'
 gem "terminal-table", '~> 1.4.2'
+gem "json", '~> 1.4.6'
 gem "net-ssh", '~> 2.0.23'
 gem "net-scp", '~> 1.0.4'
 gem "builder", '~> 2.1.2'
 gem "treetop", '~> 1.4.8'
+gem "fastthread", '~> 1.0.7'
 
 group :development do
-  gem "jeweler", '~> 1.4.0'
-  gem "thoughtbot-shoulda"
+  gem "jeweler", '~> 1.5.1'
+  gem "bundler", '~> 1.0.0'
+  gem "shoulda", ">= 0"
+  gem "rcov", ">= 0"
   gem "xml-simple", '~> 1.0.12'
-end
+  gem "ruby-debug", ">= 0"
+end

data/Gemfile.lock

@@ -2,25 +2,31 @@ GEM
   remote: http://rubygems.org/
   specs:
     builder (2.1.2)
+    columnize (0.3.1)
     commander (4.0.3)
       highline (>= 1.5.0)
-    gemcutter (0.5.0)
-      json_pure
+    fastthread (1.0.7)
     git (1.2.5)
     highline (1.6.1)
-    jeweler (1.4.0)
-      gemcutter (>= 0.1.0)
+    jeweler (1.5.1)
+      bundler (~> 1.0.0)
       git (>= 1.2.5)
-      rubyforge (>= 2.0.0)
-    json_pure (1.4.3)
+      rake
+    json (1.4.6)
+    linecache (0.43)
     net-scp (1.0.4)
       net-ssh (>= 1.99.1)
     net-ssh (2.0.23)
     polyglot (0.3.1)
-    rubyforge (2.0.4)
-      json_pure (>= 1.1.7)
+    rake (0.8.7)
+    rcov (0.9.9)
+    ruby-debug (0.10.3)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.3.0)
+    ruby-debug-base (0.10.3)
+      linecache (>= 0.3)
+    shoulda (2.11.3)
     terminal-table (1.4.2)
-    thoughtbot-shoulda (2.11.1)
     treetop (1.4.8)
       polyglot (>= 0.3.1)
     xml-simple (1.0.12)
@@ -30,11 +36,16 @@ PLATFORMS
 
 DEPENDENCIES
   builder (~> 2.1.2)
+  bundler (~> 1.0.0)
   commander (~> 4.0.3)
-  jeweler (~> 1.4.0)
+  fastthread (~> 1.0.7)
+  jeweler (~> 1.5.1)
+  json (~> 1.4.6)
   net-scp (~> 1.0.4)
   net-ssh (~> 2.0.23)
+  rcov
+  ruby-debug
+  shoulda
   terminal-table (~> 1.4.2)
-  thoughtbot-shoulda
   treetop (~> 1.4.8)
   xml-simple (~> 1.0.12)

data/README.md

@@ -16,6 +16,16 @@ Requirements
 
 * Bundler >= 1.0.0
 
+###A note on running tests###
+
+Since RuTTY is essentially a wrapper around SSH connections, a system that is capable of recieving SSH 
+connections must be available to fully test this tool. Currently, I have the tests setup to attempt to
+log into the current system via the loopback interface (localhost) as the user who ran the tests (as
+detected by `ENV['USER']`). It attempts to do this on port 22 using the public key found in
+`#{ENV['HOME']}/.ssh/id_rsa`.
+
+Note that unless all these conditions are met on your system, tests will not be fully successful.
+
 Installation
 ------------
 
@@ -45,7 +55,7 @@ After initialization, you must add nodes to the RuTTY config. This is done with
 Invoking `rutty help add_node` will give you a list of all the options to pass into it. Any options you don't pass
 will be filled in from the defaults at `$RUTTY_HOME/defaults.yaml`.
 
-	$ rutty add_node example.com -u root -k /Users/jlindsey/.ssh/id_rsa --tags example,test
+	$ rutty add_node example.com -u root -k /Users/jlindsey/.ssh/id_rsa -g example,test
 	
 The above will add a node to the RuTTY config that looks like this (in YAML):
 
@@ -68,20 +78,67 @@ can be omitted. That is to say, the following two commands are identical:
 	$ rutty dsh -a uptime
 	$ rutty -a uptime
 
-The `dsh` action can accept either a list of tags passed via `--tags` or the `-a` flag, which will run the command
+The `dsh` action can accept either a list of tags passed via `-g` or the `-a` flag, which will run the command
 on all defined nodes regardless of tags.
 
 Note that any command that has any whitespace in it must be enclosed in quotes.
 
 	$ rutty -a "free -m"
+	
+###Tags###
+
+The `rutty dsh` and `rutty scp` commands both allow the `-g` flag. The usage of this flag on these two commands
+is different than on `add_node`, where it is simply a list of tags to apply to the new node. On the remote commands,
+it essentially has three "modes": single tag, multiple comma-separated tags, and pseudo-SQL tag query.
+
+The single tag mode is simple. The command
+
+	$ rutty dsh -g foobar uptime
+
+will run the `uptime` command on every node that is tagged with "foobar".
+
+---
+
+The multiple tags mode is essentially an OR query. It will run the command on any nodes that have **ANY** of the
+specified tags. For example:
+
+	$ rutty dsh -g foo,baz,bar uptime
+
+will run the `uptime` command on any nodes that are tagged with "foo" **OR** "baz" **OR** "bar".
+
+---
+
+The pseudo-SQL tag query mode provides the most control over your tags. This allows you to pass in a string that looks
+a bit like SQL, letting you specify complex rules for your tag lookup. For example:
+
+	$ rutty dsh -g "'foo' AND 'bar'" uptime
+
+will run `uptime` on any node that has **BOTH** "foo" and "bar" tags. The command
+
+	$ rutty dsh -g "'foo' OR ('baz' AND 'bar')" uptime
+	
+will run `uptime` on any node that has a "foo" tag **OR** any node that has **BOTH** "baz" and "bar" tags.
 
 TODO
 ----
 
-* Cleanup dsh action code
-* Implement delete_node command
+* Refactor defaults config YAML to allow for a broader range of 
+  configuration options (max number of threads, default output format, etc)
+* Implement `rutty upgrade` action, which will upgrade your config files to the latest version
+
+Note on Patches/Pull Requests
+-----------------------------
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
 
 Copyright
 ---------
 
-Copyright (c) 2010 Josh Lindsey at Cloudspace. See LICENSE for details.
+Copyright (c) 2010 Josh Lindsey at Cloudspace. See LICENSE for details.
+

data/Rakefile

@@ -1,40 +1,47 @@
 require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
 require 'rake'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "rutty"
-    gem.summary = %Q{A DSH implementation in Ruby}
-    gem.description = %Q{
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  gem.name = "rutty"
+  gem.summary = %Q{A DSH implementation in Ruby}
+  gem.description = %Q{
       RuTTY is a DSH (distributed / dancer's shell) written in Ruby. It's used to run commands 
       on multiple remote servers at once, based on a tagging system. Also allows for multiple
       SCP-style uploads.
     }
-    gem.email = "josh@cloudspace.com"
-    gem.homepage = "http://github.com/jlindsey/rutty"
-    gem.authors = ["Josh Lindsey"]
-    gem.add_development_dependency "bundler", ">= 1.0.0"
-    gem.add_development_dependency "jeweler", ">= 1.4.0"
-    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
-    gem.add_development_dependency "xml-simple", ">= 1.0.12"
-    gem.add_dependency "commander", ">= 4.0.3"
-    gem.add_dependency "terminal-table", ">= 1.4.2"
-    gem.add_dependency "net-ssh", ">= 2.0.23"
-    gem.add_dependency "net-scp", ">= 1.0.4"
-    gem.add_dependency "builder", ">= 2.1.2"
-    gem.add_dependency "treetop", ">= 1.4.8"
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+  gem.email = "josh@cloudspace.com"
+  gem.homepage = "http://github.com/jlindsey/rutty"
+  gem.license = "MIT"
+  gem.authors = ["Josh Lindsey"]
+  gem.add_development_dependency "bundler", ">= 1.0.0"
+  gem.add_development_dependency "jeweler", ">= 1.5.1"
+  gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+  gem.add_development_dependency "xml-simple", ">= 1.0.12"
+  gem.add_runtime_dependency "commander", ">= 4.0.3"
+  gem.add_runtime_dependency "terminal-table", ">= 1.4.2"
+  gem.add_runtime_dependency "json", ">= 1.4.6"
+  gem.add_runtime_dependency "net-ssh", ">= 2.0.23"
+  gem.add_runtime_dependency "net-scp", ">= 1.0.4"
+  gem.add_runtime_dependency "builder", ">= 2.1.2"
+  gem.add_runtime_dependency "treetop", ">= 1.4.8"
+  gem.add_runtime_dependency "fastthread", ">= 1.0.7"
 end
+Jeweler::RubygemsDotOrgTasks.new
 
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+  test.verbose = false
 end
 
 begin
@@ -42,7 +49,7 @@ begin
   Rcov::RcovTask.new do |test|
     test.libs << 'test'
     test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
+    test.verbose = true 
   end
 rescue LoadError
   task :rcov do
@@ -50,6 +57,5 @@ rescue LoadError
   end
 end
 
-task :test => :check_dependencies
-
 task :default => :test
+

data/VERSION

@@ -1 +1 @@
-2.3.2
+2.4.0

data/bin/rutty

@@ -12,11 +12,11 @@ add_filter_options = lambda { |c|
   c.option('-k', '--keypath PATH', String, 'Path to a private key')
   c.option('-u', '--user NAME', String, 'User login')
   c.option('-p', '--port NUMBER', Integer, 'SSH port number')
-  c.option('-t', '--tags TAG1[,TAG2,...]', Array, 'Comma-separated list of tags')
+  c.option('-g', '--tags TAG1[,TAG2,...]', Array, 'Comma-separated list of tags')
 }
 
 add_remote_options = lambda { |c|
-  c.option('-t', '--tags TAG1[,TAG2,...]', String, 'Comma-separated list of tags to run the command on')
+  c.option('-g', '--tags TAG1[,TAG2,...]', String, 'Comma-separated list of tags to run the command on')
   c.option('-a', 'Run the command on ALL nodes')
   c.option('-d', '--debug', 'Enable debug output')
 }
@@ -40,7 +40,7 @@ end
 # Commander commands
 command :init do |c|
   c.syntax = "rutty init [DIR]"
-  c.summary = "Creates the default file structure for rutty"
+  c.summary = "Creates the default file structure for rutty."
   c.description = "#{c.summary} in the specified DIR. If DIR is not given, defaults to ~/.rutty"
   c.when_called do |args, options|
     args.push Rutty::Consts::CONF_DIR if args.empty?
@@ -66,9 +66,9 @@ command :add_node do |c|
 end
 
 command :list_nodes do |c|
-  # TODO: Make tag searching AND or OR, not just OR
-  c.syntax = "rutty list_nodes [options]"
-  c.summary = "List nodes according to [options]."
+  c.syntax = "rutty list_nodes [-o FORMAT]"
+  c.summary = "List all currently defined nodes."
+  c.description = "#{c.summary} Defaults to human-readable ASCII table format. Optionally specify either json or xml."
   
   c.when_called do |args, options|
     $r.list_nodes args, options
@@ -77,7 +77,7 @@ end
 
 command :dsh do |c|
   c.syntax = "rutty [dsh] [options] COMMAND"
-  c.summary = "Runs the specified COMMAND across all nodes that match [options]"
+  c.summary = "Runs the specified COMMAND across all nodes that match [options]."
   
   c.example "Get a list of all users logged in to all your web and app nodes", "rutty --tags web,app w"
   c.example "See all your nodes' current memory footprint", "rutty -a \"free -m\""
@@ -97,7 +97,7 @@ command :scp do |c|
   add_remote_options.call(c)
   
   c.when_called do |args, options|
-    $r.dsh args, options
+    $r.scp args, options
   end
 end
 

data/lib/rutty/actions.rb

@@ -25,7 +25,7 @@ module Rutty
         log "\t<%= color('exists', :cyan) %>", dir
       else
         log "\t<%= color('create', :green) %>", dir
-        Dir.mkdir dir
+        FileUtils.mkdir_p dir
       end
 
       if File.exists? general_file
@@ -137,6 +137,8 @@ module Rutty
     # @see (see #add_node)
     # @param (see #add_node)
     def dsh args, options
+      @start_time = Time.now
+      
       check_installed!
       raise Rutty::BadUsage.new "Must supply a command to run. See `rutty help dsh' for usage" if args.empty?
       raise Rutty::BadUsage.new "One of -a or --tags must be passed" if options.a.nil? and options.tags.nil?
@@ -147,24 +149,18 @@ module Rutty
         say "<%= color('No nodes defined', :yellow) %>"
         exit
       end
-      
-      HighLine.color_scheme = HighLine::SampleColorScheme.new
 
       com_str = args.pop
 
-      require 'logger'
-      require 'net/ssh'
-
       @returns = {}
-      connections = []
-
+      
       # This is necessary in order to capture exit codes and/or signals, 
       # which are't passed through when using just the ssh.exec!() semantics.
       exec_command = lambda { |ssh| 
         ssh.open_channel do |channel| 
           channel.exec(com_str) do |ch, success| 
             unless success 
-              abort "FAILED: couldn't execute command (ssh.channel.exec failure)" 
+              @returns[ssh.host][:out] = "FAILED: couldn't execute command (ssh.channel.exec failure)"
             end 
 
             channel.on_data do |ch, data|  # stdout 
@@ -177,8 +173,7 @@ module Rutty
             end 
 
             channel.on_request("exit-status") do |ch, data| 
-              exit_code = data.read_long 
-              @returns[ssh.host][:exit] = exit_code
+              @returns[ssh.host][:exit] = data.read_long
             end 
 
             channel.on_request("exit-signal") do |ch, data| 
@@ -188,25 +183,9 @@ module Rutty
         end 
         ssh.loop
       }
-
-      self.nodes.filter(options).each do |node|
-        @returns[node.host] = { :exit => 0, :out => '' }
-        begin
-          connections << Net::SSH.start(node.host, node.user, :port => node.port, :paranoid => false, 
-          :user_known_hosts_file => '/dev/null', :keys => [node.keypath], :timeout => 5,
-          :logger => Logger.new(options.debug.nil? ? $stderr : $stdout),
-          :verbose => (options.debug.nil? ? Logger::FATAL : Logger::DEBUG))
-        rescue Errno::ECONNREFUSED
-          @returns[node.host][:out] = "ERROR: Connection refused"
-          @returns[node.host][:exit] = 10000
-        rescue SocketError
-          @returns[node.host][:out] = "ERROR: no nodename nor servname provided, or not known"
-          @returns[node.host][:exit] = 20000
-        rescue Timeout::Error
-          @returns[node.host][:out] = "ERROR: Connection timeout"
-          @returns[node.host][:exit] = 30000
-        end
-      end
+      
+      @filtered_nodes = self.nodes.filter(options)
+      connections = connect_to_nodes @filtered_nodes, options.debug.nil?
 
       connections.each { |ssh| exec_command.call(ssh) }
 
@@ -215,56 +194,7 @@ module Rutty
         break if connections.empty?
       end
       
-      output = case self.output_format
-        when 'human-readable'
-          min_width = 0
-          @returns.each do |host, hash|
-            min_width = host.length if host.length > min_width
-          end
-      
-          buffer = ''
-          @returns.each do |host, hash|
-            padded_host = host.dup
-      
-            if hash[:exit] >= 10000
-              padded_host = "<%= color('#{padded_host}', :critical) %>"
-              hash[:out] = "<%= color('#{hash[:out]}', :red) %>"
-            elsif hash[:exit] > 0
-              padded_host = "<%= color('#{padded_host}', :error) %>"
-            else
-              padded_host = "<%= color('#{padded_host}', :green) %>"
-            end
-        
-            padded_host << (" " * (min_width - host.length)) if host.length < min_width
-            buffer << padded_host << "\t\t"
-        
-            buffer << hash[:out].lstrip
-          end
-          
-          buffer
-          
-        when 'json'
-          require 'json'
-          JSON.dump @returns
-          
-        when 'xml'
-          require 'builder'
-          
-          xml = Builder::XmlMarkup.new(:indent => 2)
-          
-          xml.instruct!
-          xml.nodes do
-            @returns.each do |host, hash|
-              xml.node do
-                xml.host host
-                xml.exit hash[:exit]
-                xml.out hash[:out].strip
-              end
-            end
-          end
-      end
-      
-      say output
+      say generate_output
     end
 
     ##
@@ -274,33 +204,24 @@ module Rutty
     # @see (see #add_node)
     # @param (see #add_node)
     def scp args, options
+      @start_time = Time.now
+      
       check_installed!
       raise Rutty::BadUsage.new "Must supply a local path and a remote path" unless args.length == 2
       raise Rutty::BadUsage.new "One of -a or --tags must be passed" if options.a.nil? and options.tags.nil?
       raise Rutty::BadUsage.new "Use either -a or --tags, not both" if !options.a.nil? and !options.tags.nil?
 
-      require 'logger'
-      require 'net/ssh'
-      require 'net/scp'
-      require 'pp'
-
-      connections = []
+      if self.nodes.empty?
+        say "<%= color('No nodes defined', :yellow) %>"
+        exit
+      end
 
       remote_path = args.pop
       local_path = args.pop
 
-      self.nodes.filter(options).each do |node|
-        begin
-          connections << Net::SSH.start(node.host, node.user, :port => node.port, :paranoid => false, 
-          :user_known_hosts_file => '/dev/null', :keys => [node.keypath], 
-          :logger => Logger.new(options.debug.nil? ? $stderr : $stdout),
-          :verbose => (options.debug.nil? ? Logger::FATAL : Logger::DEBUG))
-        rescue Errno::ECONNREFUSED
-          $stderr.puts "ERROR: Connection refused on #{node.host}"
-        rescue SocketError
-          $stderr.puts "ERROR: nodename nor servname provided, or not known for #{node.host}"
-        end
-      end
+      @returns = {}
+      @filtered_nodes = self.nodes.filter(options)
+      connections = connect_to_nodes @filtered_nodes, options.debug.nil?
 
       connections.each { |ssh| ssh.scp.upload! local_path, remote_path }
 
@@ -308,6 +229,158 @@ module Rutty
         connections.delete_if { |ssh| !ssh.process(0.1) { |s| s.busy? } }
         break if connections.empty?
       end
+      
+      say generate_output
+    end
+    
+    private
+    
+    ##
+    # Sets up the Net::SSH connections given a filled {Nodes} object.
+    #
+    # @since 2.4.0
+    #
+    # @see #dsh
+    # @see #scp
+    #
+    # @param [Nodes] nodes The {Node} objects to connect to
+    # @return [Array<Net:SSH::Connection>] The array of live connections
+    def connect_to_nodes nodes, debug = false
+      require 'logger'
+      require 'net/ssh'
+      require 'net/scp'
+      require 'fastthread'
+      require 'rutty/thread_pool/pool'
+      
+      connections = []
+      cons_lock = Mutex.new
+      returns_lock = Mutex.new
+      
+      pool = Rutty::ThreadPool.new 10
+      
+      nodes.each do |node|
+        pool.process do
+          returns_lock.synchronize { 
+            @returns[node.host] = { :exit => 0, :out => '' } 
+          }
+          begin
+            cons_lock.synchronize {
+              connections << Net::SSH.start(node.host, node.user, :port => node.port, :paranoid => false, 
+              :user_known_hosts_file => '/dev/null', :keys => [node.keypath], :timeout => 5,
+              :logger => Logger.new(debug ? $stderr : $stdout),
+              :verbose => (debug ? Logger::FATAL : Logger::DEBUG))
+            }
+          rescue Errno::ECONNREFUSED
+            returns_lock.synchronize {
+              @returns[node.host][:out] = "ERROR: Connection refused"
+              @returns[node.host][:exit] = 10000
+            }
+            
+            cons_lock.synchronize { connections << nil }
+          rescue SocketError
+            returns_lock.synchronize {
+              @returns[node.host][:out] = "ERROR: no nodename nor servname provided, or not known"
+              @returns[node.host][:exit] = 20000
+            }
+            
+            cons_lock.synchronize { connections << nil }
+          rescue Timeout::Error
+            returns_lock.synchronize {
+              @returns[node.host][:out] = "ERROR: Connection timeout"
+              @returns[node.host][:exit] = 30000
+            }
+            
+            cons_lock.synchronize { connections << nil }
+          rescue Net::SSH::AuthenticationFailed => e
+            returns_lock.synchronize {
+              @returns[node.host][:out] = "ERROR: Authentication failure (#{e.to_s})"
+              @returns[node.host][:exit] = 40000
+            }
+            
+            cons_lock.synchronize { connections << nil }
+          end
+        end
+      end
+      
+      loop do
+        break if nodes.length == connections.length
+        sleep 0.2
+      end
+      
+      pool.shutdown
+      connections.compact
+    end
+    
+    ##
+    # Generates the output of the remote actions, based on the value of {Rutty::Runner#output_format}.
+    #
+    # @since 2.4.0
+    #
+    # @see #dsh
+    # @see #scp
+    #
+    # @return [String] The formatted string to output
+    def generate_output
+      case self.output_format
+      when 'human-readable'
+        HighLine.color_scheme = HighLine::SampleColorScheme.new
+        
+        min_width = 0
+        @returns.each do |host, hash|
+          min_width = host.length if host.length > min_width
+        end
+
+        buffer = ''
+        @returns.each do |host, hash|
+          padded_host = host.dup
+
+          if hash[:exit] >= 10000
+            padded_host = "<%= color('#{padded_host}', :critical) %>"
+            hash[:out] = "<%= color('#{hash[:out]}', :red) %>"
+          elsif hash[:exit] > 0
+            padded_host = "<%= color('#{padded_host}', :error) %>"
+          else
+            padded_host = "<%= color('#{padded_host}', :green) %>"
+          end
+
+          padded_host << (" " * (min_width - host.length)) if host.length < min_width
+          buffer << padded_host << "\t\t"
+
+          buffer << hash[:out].lstrip
+        end
+
+        @end_time = Time.now
+
+        total_nodes = @filtered_nodes.length
+        total_error_nodes = @returns.reject { |host, hash| hash[:exit] == 0 }.length
+        total_time = @end_time - @start_time
+
+        buffer.rstrip!
+        buffer << "\n\n" << "<%= color('#{total_nodes} host(s), #{total_error_nodes} error(s), #{seconds_in_words total_time}', "
+        buffer << (total_error_nodes > 0 ? ":red" : ":green") << ") %>"
+
+        buffer
+
+      when 'json'
+        require 'json'
+        JSON.dump @returns
+
+      when 'xml'
+        require 'builder'
+
+        xml = Builder::XmlMarkup.new(:indent => 2)
+
+        xml.instruct!
+        xml.nodes do
+          @returns.each do |host, hash|
+            xml.node do
+              xml.host host
+              xml.exit hash[:exit]
+              xml.out hash[:out].strip
+            end
+          end
+        end
+      end
     end
   end
 end