remote_run 0.1.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rvmrc

@@ -0,0 +1,3 @@
+export rvm_path="/Users/pivotal/.rvm"
+rvm_gemset_create_on_use_flag=1
+rvm_install_on_use_flag=1

data/Gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gem "highline"
+gem "rspec"
+gem "rake"
+
+gemspec

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2011 Case Commons, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/Readme.md

@@ -0,0 +1,79 @@
+# Remote Run.
+
+Our development team wanted a way to distribute our test suite to many machines
+to reduce the total run time.  Remote Run is intended to be a simple way to 
+run a list of shell scripts on a pool of hosts until all have completed. 
+
+When two Remote Runs are in progress, the runners will compete to lock
+machines until all tasks are complete.
+
+
+## Example: 
+
+        require 'rubygems'
+        require 'remote_run'
+
+        hosts = ["broadway", "wall"]
+        setup = "source ~/.profile; rvm use ree; bundle install;"
+        tasks = [
+          "#{setup} bundle exec rspec spec/models",
+          "#{setup} bundle exec rspec spec/controllers"
+        ]
+
+        # configure runner
+        runner = Runner.new do |config|
+          config.hosts = hosts
+          config.tasks = tasks
+        end
+
+        # kick off the run
+        runner.run
+
+
+## Configuration Options:
+
+        Required:
+        hosts - hostnames of remote machines.
+        tasks - a string that is a shell script to be run on one of the hosts.
+
+        Optional:
+        local_path - the local path to be rsync'd  (default: working directory)
+        temp_path - the location where the working directory is cached on the local machine when starting a run (default: /tmp/remote)
+        remote_path - the location to rsync files to on the remote host.  (default: /tmp/remote)
+        exclude - directories to exclude when rsyncing to remote host (default: [])
+        login_as - the user used to log into ssh (default: current user)
+
+
+## Accessible Attributes:
+
+        local_hostname - your computer's hostname
+        identifier - a unique identifier for your test run
+
+
+## What it Does:
+
+* checks that all hosts can be logged into via ssh
+* runs each task on a remote host in parallel
+  * finds an unlocked remote host
+  * locks a remote host (puts a file on the remote host)
+  * finds a task to run
+  * forks a separate process and gives it the selected task
+    - rsyncs your current directory to the locked remote host
+    - via ssh, runs a shell command of your choice on the locked remote host
+    - unlocks the machine (removes the file from the remote host)
+    - returns the status code of the shell script
+  * finds the next machine to be locked
+  * waits for all forks to return
+  * displays success message if all status codes from the forks are zero (0)
+  * displays failure message if any status codes from the forks are non-zero
+
+
+Dependencies:
+----------------------------------------------------------------------
+* HighLine 
+
+
+License:
+----------------------------------------------------------------------
+
+MIT

data/examples/demo-remote-run

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'remote_run'
+
+Runner.log("Scanning 10.64.20.* for hosts...", :yellow)
+#hosts = `nmap -p 22 10.64.20.1/24 --open -sV | grep -B3 "Debian" | grep 10.64.20 | cut -f 5 -d " "`.lines.map(&:strip)
+hosts = %w{ 10.64.20.1 10.64.20.12 10.64.20.15 10.64.20.18 10.64.20.21 10.64.20.26 10.64.20.30 10.64.20.38 10.64.20.43 10.64.20.47 10.64.20.49 10.64.20.50 10.64.20.51 10.64.20.53 10.64.20.56 10.64.20.58 10.64.20.59 10.64.20.61 10.64.20.62 10.64.20.64 10.64.20.66 10.64.20.69 10.64.20.74 10.64.20.76 10.64.20.79 10.64.20.80 10.64.20.93 10.64.20.101 10.64.20.110 10.64.20.111 10.64.20.122 10.64.20.149 10.64.20.164 10.64.20.168 10.64.20.198 10.64.20.209 10.64.20.214 10.64.20.221 10.64.20.244 10.64.20.245 10.64.20.246 10.64.20.247 10.64.20.248 10.64.20.249 10.64.20.250 10.64.20.251 10.64.20.252 10.64.20.253 10.64.20.254 } 
+Runner.log("With all tasks passing...")
+tasks = []
+50.times do |n|
+  tasks << "sleep 2; date;"
+end
+tasks << "cat /foo/bar"
+
+Runner.run do |config|
+  config.tasks = tasks
+  config.login_as = "pivotalcb"
+  config.hosts = hosts
+end
+
+Runner.log("Expected one failure to test that it still works...")

data/lib/remote_run/host.rb

@@ -0,0 +1,130 @@
+class Host
+  FAIL = 1
+  PASS = 0
+  SSH_CONFIG = " -o ControlMaster=auto -o ControlPath=~/.ssh/master-%l-%r@%h:%p -o NumberOfPasswordPrompts=0 -o StrictHostKeyChecking=no -4 "
+  attr_reader :hostname
+  attr_reader :lock_file
+
+  def initialize(hostname)
+    @hostname = hostname
+    @lock_file = LockFile.new(@hostname, $runner.local_hostname, $runner.identifier)
+  end
+
+  def lock
+    unless locked?
+      @lock_file.get && locked_by_me?
+    end
+  end
+
+  def unlock
+    @lock_file.release
+  end
+
+  def run(task)
+    Runner.log("Running '#{task}' on #{@hostname}", :white)
+    command = %Q{ssh #{SSH_CONFIG} #{ssh_host_and_user} 'cd #{$runner.remote_path}; #{task}' 2>&1}
+    system(command)
+    $?.exitstatus
+  end
+
+  def copy_codebase
+    Runner.log("Copying from #{$runner.temp_path} to #{@hostname}:#{$runner.remote_path}", :yellow)
+    system("ssh #{SSH_CONFIG} #{ssh_host_and_user} 'mkdir -p #{$runner.remote_path}'")
+    excludes = $runner.exclude.map { |dir| "--exclude '#{dir}'"}
+    if system(%{rsync --delete --delete-excluded #{excludes.join(" ")} --rsh='ssh #{SSH_CONFIG}' --timeout=60 -a #{$runner.temp_path}/ #{ssh_host_and_user}:#{$runner.remote_path}/})
+      Runner.log("Finished copying to #{@hostname}", :green)
+      return true
+    else
+      Runner.log("rsync failed on #{@hostname}.", :red)
+      return false
+    end
+  end
+
+  def is_up?
+    result = `ssh #{SSH_CONFIG} -o ConnectTimeout=2 #{ssh_host_and_user} "echo 'success'" 2>/dev/null`.strip
+    if result == "success"
+      Runner.log("#{@hostname} is up", :green)
+      return true
+    else
+      Runner.log("#{@hostname} is down: #{result}", :red)
+      return false
+    end
+  end
+
+  def start_ssh_master_connection
+    system("ssh #{SSH_CONFIG} #{ssh_host_and_user} -M &> /dev/null")
+  end
+
+  private
+
+  def ssh_host_and_user
+    "#{$runner.login_as}@#{@hostname}"
+  end
+
+  def locked?
+    @lock_file.locked?
+  end
+
+  def locked_by_me?
+    @lock_file.locked_by_me?
+  end
+
+  class LockFile
+    FILE = "/tmp/remote-run-lock"
+
+    def initialize(remote_hostname, local_hostname, unique_run_marker)
+      @filename = FILE
+      @locker = "#{local_hostname}-#{unique_run_marker}"
+      @remote_file = RemoteFile.new(remote_hostname)
+    end
+
+    def release
+      if locked_by_me?
+        @remote_file.delete(@filename)
+      end
+    end
+
+    def locked?
+      @remote_file.exist?(@filename)
+    end
+
+    def locked_by_me?
+      @remote_file.exist?(@filename) && @remote_file.read(@filename).strip == @locker
+    end
+
+    def get
+      @remote_file.write(@filename, @locker)
+    end
+
+    class RemoteFile
+      def initialize(hostname)
+        @hostname = hostname
+      end
+
+      def exist?(file_path)
+        run_and_test("test -f #{file_path}")
+      end
+
+      def read(file_path)
+        run("test -e #{file_path} && cat #{file_path}")
+      end
+
+      def write(file_path, text)
+        run_and_test("test -e #{file_path} || echo #{text} > #{file_path}")
+      end
+
+      def delete(file_path)
+        run_and_test("rm -f #{file_path}")
+      end
+
+      def run(command)
+        `ssh #{Host::SSH_CONFIG} #{$runner.login_as}@#{@hostname} '#{command};'`.strip
+      end
+
+      def run_and_test(command)
+        system("ssh #{Host::SSH_CONFIG} #{$runner.login_as}@#{@hostname} '#{command}' 2>/dev/null")
+      end
+    end
+  end
+end
+

data/lib/remote_run/runner.rb

@@ -0,0 +1,233 @@
+class Runner
+  attr_accessor :remote_path, :local_path, :login_as, :exclude, :temp_path
+  attr_reader :local_hostname, :identifier
+  @@start_time = Time.now
+  @@stty_config = `stty -g`
+
+  def initialize
+    @task_manager = TaskManager.new
+    @host_manager = HostManager.new
+
+    # config options
+    @local_path = Dir.getwd
+    @login_as = `whoami`.strip
+    @remote_path = "/tmp/remote"
+    @exclude = []
+    @temp_path = "/tmp/remote"
+
+    # used in the runner
+    @identifier = `echo $RANDOM`.strip
+    @local_hostname = `hostname`.strip
+    @results = []
+    @children = []
+    @failed = []
+    @last_timestamp = Time.now.strftime("%S")[0]
+
+    $runner = self
+    yield self
+  end
+
+  def self.run(&block)
+    @@start_time = Time.now
+    runner = new(&block)
+    runner.run
+  end
+
+  def self.run_time
+    minutes = ((Time.now - @@start_time) / 60).to_i
+    seconds = ((Time.now - @@start_time) % 60).to_i
+    "#{minutes}:#{"%02d" % seconds}"
+  end
+
+  def self.log(message, color = :yellow)
+    highline = HighLine.new
+    system("stty #{@@stty_config} 2>/dev/null")
+    highline.say(highline.color("[Remote :: #{$runner.identifier} :: #{run_time}] #{message}", color))
+  end
+
+  def hosts
+    @host_manager.all
+  end
+
+  def hosts=(hostnames)
+    hostnames.each do |hostname|
+      @host_manager.add(hostname)
+    end
+  end
+
+  def tasks=(shell_commands)
+    shell_commands.each do |shell_command|
+      @task_manager.add(shell_command)
+    end
+  end
+
+  def run
+    @host_manager.unlock_on_exit
+    @host_manager.start_ssh_master_connections
+    sync_working_copy_to_temp_location
+    hosts = []
+
+    Runner.log("Starting tasks... #{Time.now}")
+
+    @starting_number_of_tasks = @task_manager.count
+    while @task_manager.has_more_tasks?
+      hosts = @host_manager.hosts.dup if hosts.empty?
+
+      display_log
+      check_for_finished
+
+      if host = hosts.sample
+        hosts.delete(host)
+        if host.lock
+          task = @task_manager.find_task
+          @children << fork do
+            begin
+              this_host = host.dup
+              unless this_host.copy_codebase
+                @task_manager.add(task)
+                status = 0
+              end
+              status = this_host.run(task)
+              host.unlock
+              Runner.log("#{host.hostname} failed.", :red) if status != 0
+            rescue Errno::EPIPE
+              Runner.log("broken pipe on #{host.hostname}...")
+            ensure
+              Process.exit!(status)
+            end
+          end
+        end
+      end
+    end
+
+    Runner.log("All tasks started... #{Time.now}")
+
+    while @children.length > 0
+      display_log
+      check_for_finished
+    end
+
+    failed_tasks = @results.select { |result| result != 0 }
+    status_code = if failed_tasks.length == 0
+      Runner.log("Task passed.", :green)
+      Host::PASS
+    else
+      Runner.log("#{failed_tasks.length} task(s) failed.", :red)
+      Host::FAIL
+    end
+
+    Runner.log("Total Time: #{self.class.run_time} minutes.")
+    status_code
+  end
+
+  def check_for_finished
+    @children.each do |child_pid|
+      if Process.waitpid(child_pid, Process::WNOHANG)
+        if $?.exitstatus != 0
+          @failed << child_pid
+        end
+
+        @results << $?.exitstatus
+        @children.delete(child_pid)
+      end
+    end
+    sleep(0.5)
+  end
+
+  private
+
+  def sync_working_copy_to_temp_location
+    Runner.log("Creating temporary copy of #{@local_path} in #{@temp_path}...")
+    excludes = exclude.map { |dir| "--exclude '#{dir}'"}
+    system("rsync --delete --delete-excluded #{excludes.join(" ")} -aq #{@local_path}/ #{@temp_path}/")
+    Runner.log("Done.")
+  end
+
+  def display_log
+    now = Time.now.strftime("%S")[0]
+    unless now == @last_timestamp
+      display_status("Waiting on #{@task_manager.count} of #{@starting_number_of_tasks} tasks to start.") if @task_manager.count > 0
+      display_status("Waiting on #{@children.length} of #{@starting_number_of_tasks - @task_manager.count} started tasks to finish. #{@failed.size} failed.") if @children.length > 0
+      $stdout.print("\n\n")
+      $stdout.flush
+      @last_timestamp = now
+    end
+  end
+
+  def display_status(message)
+    Runner.log(message, :yellow)
+  end
+
+  class HostManager
+    def initialize(&block)
+      @hosts = []
+    end
+
+    def all
+      @hosts
+    end
+
+    def add(hostname)
+      host = Host.new(hostname)
+      Thread.new do
+        if host.is_up?
+          @hosts << host
+        end
+      end
+    end
+
+    def hosts
+      while @hosts.empty?
+        Runner.log("Waiting for hosts...")
+        sleep(0.5)
+      end
+
+      @hosts
+    end
+
+    def unlock_on_exit
+      at_exit do
+        all.each do |host|
+          begin
+            host.unlock
+          rescue Errno::EPIPE
+          end
+        end
+      end
+    end
+
+    def start_ssh_master_connections
+      all.each do |host|
+        fork do
+          host.start_ssh_master_connection
+        end
+      end
+    end
+  end
+
+  class TaskManager
+    def initialize
+      @tasks = []
+    end
+
+    def add(script)
+      @tasks.push(script)
+    end
+
+    def find_task
+      @tasks.shift
+    end
+
+    def all
+      @tasks
+    end
+
+    def count
+      @tasks.length
+    end
+
+    def has_more_tasks?
+      @tasks.size > 0
+    end
+  end
+end

data/lib/remote_run/version.rb

@@ -0,0 +1,3 @@
+module RemoteRun
+  VERSION = "0.1.0"
+end

data/lib/remote_run.rb

@@ -0,0 +1,3 @@
+require File.join(File.dirname(__FILE__), "remote_run", "host")
+require File.join(File.dirname(__FILE__), "remote_run", "runner")
+require 'highline'

data/remote_run.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "remote_run/version"
+
+Gem::Specification.new do |s|
+  s.name        = "remote_run"
+  s.version     = RemoteRun::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Case Commons, LLC"]
+  s.email       = ["casecommons-dev@googlegroups.com"]
+  s.homepage    = "https://github.com/Casecommons/remote_run"
+  s.summary     = %q{Run N shell scripts on a pool of remote hosts}
+  s.description = %q{Can be used as a parallel unit test runner}
+
+  s.rubyforge_project = "remote_run"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_runtime_dependency("highline")
+  s.add_development_dependency('rspec')
+end

data/spec/remote_run/host_spec.rb

@@ -0,0 +1,108 @@
+require 'spec_helper'
+
+describe Host do
+  context "when locking" do
+    let(:host) { Host.new("localhost") }
+
+    it "can be locked" do
+      host.lock.should be_true
+    end
+
+    it "cannot be locked twice" do
+      host.lock.should be_true
+      host.lock.should be_false
+    end
+
+    it "tells the lock file to get a lock" do
+      host.lock
+      `ssh localhost 'cat /tmp/remote-run-lock'`.strip.should == "#{$runner.local_hostname}-#{$runner.identifier}"
+    end
+  end
+
+  context "when locked by someone else" do
+    before { lock_file.get }
+    let(:host) { Host.new("localhost") }
+    let(:lock_file) {
+      lock_file = Host::LockFile.new("localhost", "myfakelocalhost", "999")
+    }
+
+    it "cannot be unlocked by me" do
+      host.unlock.should be_false
+    end
+  end
+
+  context "when locked by me" do
+    before { host.lock }
+    let(:host) { Host.new("localhost") }
+
+    it "cannot be locked" do
+      host.lock.should be_false
+    end
+
+    it "can be unlocked" do
+      host.unlock.should be_true
+    end
+
+    it "removes a file on the remote filesystem to unlock" do
+      `ssh localhost 'test -e /tmp/remote-run-lock'; echo $?;`.strip.should == "0"
+      host.unlock
+      `ssh localhost 'test -e /tmp/remote-run-lock'; echo $?;`.strip.should == "1"
+    end
+  end
+
+  context "when checking to see if a host is up" do
+    context "when using an authorized host" do
+      let(:host) { Host.new("localhost") }
+
+      it "returns true" do
+        host.is_up?.should be_true
+      end
+    end
+
+    context "when using an unauthorized host" do
+      let(:host) { Host.new("foozmcbarry") }
+
+      it "returns false" do
+        host.is_up?.should be_false
+      end
+    end
+  end
+
+  context "when running a task" do
+    before do
+      `ssh localhost 'rm -rf /tmp/testing-remote-run'`
+      host.lock
+    end
+
+    let(:host) { Host.new("localhost") }
+
+    context "when executing a shell command with a zero status code" do
+      it "returns zero" do
+        host.run("date > /dev/null").should == 0
+      end
+    end
+
+    context "when executing a shell command with a non-zero status code" do
+      it "returns non-zero status code" do
+        host.run("cat /foo/bar 2>/dev/null").should_not == 0
+      end
+    end
+  end
+
+  describe "#copy_codebase" do
+    before do
+      `ssh localhost 'rm -rf /tmp/testing-remote-run'`
+      host.lock
+    end
+
+    let(:host) { Host.new("localhost") }
+
+    it "copies the codebase to a remote directory" do
+      $runner.remote_path = "/tmp/testing-remote-run"
+      `ssh localhost 'test -e /tmp/testing-remote-run'; echo $?`.strip.should_not == "0"
+      host.copy_codebase
+      `ssh localhost 'test -e /tmp/testing-remote-run'; echo $?`.strip.should == "0"
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'rspec'
+require Dir.pwd + '/lib/remote_run'
+
+Runner.new do |config|
+  config.tasks = []
+  config.hosts = []
+end
+
+RSpec.configure do |config|
+  config.after(:each) do
+    system("rm -f #{Host::LockFile::FILE}")
+  end
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: remote_run
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Case Commons, LLC
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-11 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: &2151825780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2151825780
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2151824680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2151824680
+description: Can be used as a parallel unit test runner
+email:
+- casecommons-dev@googlegroups.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- LICENSE
+- Rakefile
+- Readme.md
+- examples/demo-remote-run
+- lib/remote_run.rb
+- lib/remote_run/host.rb
+- lib/remote_run/runner.rb
+- lib/remote_run/version.rb
+- remote_run.gemspec
+- spec/remote_run/host_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/Casecommons/remote_run
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4543623679604844651
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4543623679604844651
+requirements: []
+rubyforge_project: remote_run
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Run N shell scripts on a pool of remote hosts
+test_files:
+- spec/remote_run/host_spec.rb
+- spec/spec_helper.rb