weave 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+Gemfile.lock
+/.vagrant
+/.yardoc/
+/doc/

data/.yardopts

@@ -0,0 +1 @@
+--no-private --protected --markup=markdown -- lib/**/*.rb

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in weave.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Caleb Spare
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# Weave
+
+Simple parallel ssh.
+
+## Implemented features
+
+* Connection caching
+* Parallel execution (with thread/connection limit)
+* Serial execution
+* Commands:
+
+  - `run`
+
+## Other ideas
+
+* Commands:
+
+  - `sudo`
+  - `rsync`
+  - `put`?
+  - `get`?
+
+* Handle password prompts (maybe just sudo prompt)
+* Handle ctrl-c correctly?
+
+## To do:
+
+* Full readme with detailed usage

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task :test => ["test:integrations"]
+
+namespace :test do
+  Rake::TestTask.new(:integrations) do |task|
+    task.test_files = FileList["test/integrations/**/*.rb"]
+  end
+end

data/Vagrantfile

@@ -0,0 +1,21 @@
+Vagrant::Config.run do |config|
+  def apply_common_config_options(c)
+    c.vm.box = "lucid64"
+    c.vm.provision :shell, :inline =>
+        "sudo mkdir -p /root/.ssh && sudo cp /home/vagrant/.ssh/authorized_keys /root/.ssh/"
+  end
+
+  config.vm.define :weave1 do |config|
+    config.vm.host_name = "weave1"
+    config.vm.network :hostonly, "1.2.3.90"
+    config.vm.forward_port 22, 3220 # ssh
+    apply_common_config_options(config)
+  end
+
+  config.vm.define :weave2 do |config|
+    config.vm.host_name = "weave2"
+    config.vm.network :hostonly, "1.2.3.91"
+    config.vm.forward_port 22, 3221 #ssh
+    apply_common_config_options(config)
+  end
+end

data/examples/basic.rb

@@ -0,0 +1,7 @@
+$:.unshift File.join(File.dirname(__FILE__), "../lib")
+
+require "weave"
+
+pool = Weave.connect(ARGV) do
+  run "ls"
+end

data/examples/parallel-console.rb

@@ -0,0 +1,26 @@
+$:.unshift File.join(File.dirname(__FILE__), "../lib")
+
+require "readline"
+require "weave"
+
+usage = <<-EOS
+Usage:
+  $ ./parallel-ssh user@host1 user@host2 ...
+EOS
+
+abort usage if ARGV.empty?
+
+$stty_state = `stty -g`.chomp
+$pool = Weave.connect(ARGV)
+
+while command = Readline.readline(">>> ", true)
+  break unless command # ctrl-D
+  command.chomp!
+  next if command.empty?
+  break if ["exit", "quit"].include? command
+  $pool.execute { run command }
+end
+
+$pool.disconnect!
+`stty #{$stty_state}`
+puts "Bye."

data/lib/weave.rb

@@ -0,0 +1,171 @@
+require "net/ssh"
+require "thread"
+require "singleton"
+
+module Weave
+  DEFAULT_THREAD_POOL_SIZE = 10
+
+  # @private
+  COLORS = {
+    red: 1,
+    green: 2,
+    yellow: 3,
+    blue: 4,
+    magenta: 5,
+    cyan: 6,
+    white: 7,
+    default: 8
+  }
+
+  # Create a connection pool for an array of hosts. Each host must have a user specified (e.g.,
+  # root@example.com). If a block is given, then the options are passed through to the underlying
+  # ConnectionPool and the block is immediately run in the context of each connection. Otherwise, a pool is
+  # returned.
+  #
+  # @see ConnectionPool#execute
+  def self.connect(host_list, options = {}, &block)
+    pool = ConnectionPool.new(host_list)
+    if block_given?
+      pool.execute(options, &block)
+      pool.disconnect!
+    else
+      return pool
+    end
+  end
+
+  # @private
+  def self.color_string(string, color) "\e[01;#{COLORS[color]+30}m#{string}\e[m" end
+
+  # Spread work, identified by a key, across multiple threads.
+  # @private
+  def self.with_thread_pool(keys, thread_pool_size, &block)
+    work_queue = Queue.new
+    mutex = Mutex.new
+    keys.each { |key| work_queue << key }
+
+    threads = (1..thread_pool_size).map do |i|
+      Thread.new do
+        begin
+          while (key = work_queue.pop(true))
+            yield key, mutex
+          end
+        rescue ThreadError # Queue is empty
+        end
+      end
+    end
+    threads.each(&:join)
+    nil
+  end
+
+  # A pool of SSH connections. Operations over the pool may be performed in serial or in parallel.
+  class ConnectionPool
+    # @param [Array] host_list the array of hosts, of the form user@host
+    def initialize(host_list)
+      @connections = host_list.reduce({}) { |pool, host| pool.merge(host => LazyConnection.new(host)) }
+    end
+
+    # Run a command over the connection pool. The block is evaluated in the context of LazyConnection.
+    #
+    # @param [Hash] options the various knobs
+    # @option options [Fixnum] :num_threads the number of concurrent threads to use to process this command.
+    #     Defaults to `DEFAULT_THREAD_POOL_SIZE`.
+    # @option options [Boolean] :serial whether to process the command for each connection one at a time.
+    # @option options [Fixnum] :batch_by if set, group the connections into batches of no more than this value
+    #     and fully process each batch before starting the next one.
+    def execute(options = {}, &block)
+      options[:num_threads] ||= DEFAULT_THREAD_POOL_SIZE
+      if options[:serial]
+        @connections.each_key { |host| @connections[host].self_eval &block }
+      elsif options[:batch_by]
+        @connections.each_key.each_slice(options[:batch_by]) do |batch|
+          Weave.with_thread_pool(batch, options[:num_threads]) do |host, mutex|
+            @connections[host].self_eval mutex, &block
+          end
+        end
+      else
+        Weave.with_thread_pool(@connections.keys, options[:num_threads]) do |host, mutex|
+          @connections[host].self_eval mutex, &block
+        end
+      end
+    end
+
+    # Disconnect all open connections.
+    def disconnect!() @connections.each_value(&:disconnect) end
+  end
+
+  # @private
+  class NilMutex
+    include Singleton
+    def synchronize() yield end
+  end
+
+  # An SSH connection which isn't established until it's needed.
+  class LazyConnection
+    # The username for the ssh connection
+    attr_reader :user
+    # The hostname for the ssh connection
+    attr_reader :host
+
+    # @param [String] host_string the host of the form user@host
+    def initialize(host_string)
+      @user, @host = self.class.user_and_host(host_string)
+      @connection = nil
+      @mutex = NilMutex.instance
+    end
+
+    # Run a command on this connection. This will open a connection if it's not already connected. The way the
+    # output is presented is determined by the option `:output`. The default, `:output => :pretty`, prints
+    # each line of output with the name of the host and whether the output is stderr or stdout. If `:output =>
+    # :raw`, then the output will be passed as is directly back to `STDERR` or `STDOUT` as appropriate. If
+    # `:output => :capture`, then this method returns the output in a hash of the form
+    # `{ :stdout => [...], :stderr => [...] }`.
+    #
+    # @param [Hash] options the output options
+    # @option options [Symbol] :output the output format
+    def run(command, options = {})
+      options[:output] ||= :pretty
+      @connection ||= Net::SSH.start(@host, @user)
+      output = { :stderr => [], :stdout => [] }
+      @connection.exec(command) do |channel, stream, data|
+        case options[:output]
+        when :capture
+          output[stream] << data
+        when :raw
+          out_stream = stream == :stdout ? STDOUT : STDERR
+          out_stream.print data
+        else
+          stream_colored = case stream
+                           when :stdout then Weave.color_string("out", :green)
+                           when :stderr then Weave.color_string("err", :red)
+                           end
+          lines = data.split("\n").map { |line| "[#{stream_colored}|#{host}] #{line}" }.join("\n")
+            @mutex.synchronize { puts lines }
+        end
+      end
+      @connection.loop(0.1)
+      (options[:output] == :capture) ? output : nil
+    end
+
+    # @private
+    def self.user_and_host(host_string)
+      user, at, host = host_string.rpartition("@")
+      if [user, host].any? { |part| part.nil? || part.empty? }
+        raise "Bad hostname (needs to be of the form user@host): #{host_string}"
+      end
+      [user, host]
+    end
+
+    # @private
+    def self_eval(mutex = nil, &block)
+      @mutex = mutex || NilMutex.instance
+      instance_eval &block
+      @mutex = NilMutex.instance
+    end
+
+    # Disconnect, if connected.
+    def disconnect()
+      @connection.close if @connection
+      @connection = nil
+    end
+  end
+end

data/lib/weave/version.rb

@@ -0,0 +1,3 @@
+module Weave
+  VERSION = "0.0.1"
+end

data/test/integrations/sanity_test.rb

@@ -0,0 +1,75 @@
+require File.expand_path(File.join(File.dirname(__FILE__), "../test_helper"))
+
+require "net/ssh"
+require "weave"
+
+class SanityTest < Scope::TestCase
+  TEST_HOSTS = [1, 2].map { |i| "weave#{i}" }
+  ROOT_AT_TEST_HOSTS = TEST_HOSTS.map { |host| "root@#{host}" }
+
+  setup_once do
+    # Make sure the machines are up.
+    vagrant_status = `bundle exec vagrant status`
+    unless $?.to_i.zero? && TEST_HOSTS.each { |host| vagrant_status =~ /#{host}\w+running/ }
+      abort "You need to set up the test vagrant virtual machines to run the sanity test." \
+            "Run 'bundle exec vagrant up'."
+    end
+    # Make sure the user's ssh config has weave entries.
+    TEST_HOSTS.each do |host|
+      config = Net::SSH::Config.load("~/.ssh/config", host)
+      unless config["hostname"] == "127.0.0.1"
+        abort "You need to add weave{1,2} to your ~/.ssh/config." \
+              "You can use the output of 'bundle exec vagrant ssh-config weave1'"
+      end
+    end
+  end
+
+  context "executing some commands" do
+    should "run some simple commands" do
+      output = Hash.new { |h, k| h[k] = [] }
+      Weave.connect(ROOT_AT_TEST_HOSTS) do
+        output[host] = run("echo 'hello'", :output => :capture)
+      end
+      TEST_HOSTS.each do |host|
+        assert_empty output[host][:stderr]
+        assert_equal ["hello\n"], output[host][:stdout]
+      end
+    end
+
+    context "in serial" do
+      should "run some commands in the expected order" do
+        output = []
+        Weave.connect(ROOT_AT_TEST_HOSTS, :serial => true) do
+          command = (host == "weave1") ? "sleep 0.2; echo 'delayed'" : "echo 'on time'"
+          output += run(command, :output => :capture)[:stdout]
+        end
+        assert_equal ["delayed\n", "on time\n"], output
+      end
+    end
+
+    context "in parallel" do
+      should "run some commands in the expected order" do
+        output = []
+        Weave.connect(ROOT_AT_TEST_HOSTS) do
+          command = (host == "weave1") ? "sleep 0.2; echo 'delayed'" : "echo 'on time'"
+          result = run(command, :output => :capture)
+          output += result[:stdout]
+        end
+        assert_equal ["on time\n", "delayed\n"], output
+      end
+    end
+
+    context "on a connection pool" do
+      should "run basic commands" do
+        output = Hash.new { |h, k| h[k] = [] }
+        Weave.connect(ROOT_AT_TEST_HOSTS).execute do
+          output[host] = run("echo 'hello'", :output => :capture)
+        end
+        TEST_HOSTS.each do |host|
+          assert_empty output[host][:stderr]
+          assert_equal ["hello\n"], output[host][:stdout]
+        end
+      end
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+$:.unshift File.join(File.dirname(__FILE__), "../lib")
+
+require "scope"
+require "minitest/autorun"

data/weave.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/weave/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Caleb Spare"]
+  gem.email         = ["cespare@gmail.com"]
+  gem.description   = %q{Simple parallel ssh.}
+  gem.summary       = %q{Simple parallel ssh.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "weave"
+  gem.require_paths = ["lib"]
+  gem.version       = Weave::VERSION
+
+  # For running integration tests.
+  gem.add_development_dependency "vagrant"
+  gem.add_development_dependency "scope"
+  gem.add_development_dependency "rake"
+  # For generating the docs
+  gem.add_development_dependency "yard"
+  gem.add_development_dependency "redcarpet"
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: weave
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Caleb Spare
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: vagrant
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: scope
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple parallel ssh.
+email:
+- cespare@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .yardopts
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- Vagrantfile
+- examples/basic.rb
+- examples/parallel-console.rb
+- lib/weave.rb
+- lib/weave/version.rb
+- test/integrations/sanity_test.rb
+- test/test_helper.rb
+- weave.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Simple parallel ssh.
+test_files:
+- test/integrations/sanity_test.rb
+- test/test_helper.rb
+has_rdoc: