clusterfuck 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Trevor Fountain
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,49 @@
+= Clusterfuck
+==== A Subversive Distributed-Systems Tool
+
+Clusterfuck is a tool for automating the process of SSH-ing into remote machines and kickstarting a large number
+of jobs. It's probably best explained by an example, so here's what I use it for:
+
+As part of my research I need to compute the distance between each pair of objects in a set of about 70,000 items.
+Computing the distance between each pair takes a few seconds; running the entire job on a single machine generally takes over a day.
+However, as a member of the University I have a ssh login that works on quite a few machines, so I found myself breaking the job up into smaller, quicker chunks and running each chunk on a different machine.
+Clusterfuck was born out of my frustration with that method -- "surely," I said to myself, "this can be automated."
+
+If you have a lot of jobs to run and access to multiple machines on which to run them, Clusterfuck is for you!
+
+== Usage
+To use Clusterfuck you'll first need to create a configuration file (a "clusterfile"). An example clusterfile might look something like this:
+  
+  Clusterfuck::Task.new do |task|
+    task.hosts = %w{clark asimov}
+    task.jobs = (0..3).map { |x| Clusterfuck::Job.new("host{x}","sleep 0.5 && hostname") }
+    task.temp = "fragments"
+    task.username = "SSHUSERNAME"
+    task.password = "SSHPASSWORD"
+    task.debug = true
+  end
+
+This creates a new clusterfuck task and distributes the jobs across two hosts, +clark+ and +asimov+. 
+The jobs to be run in this case are pretty trivial; we basically ssh into each machine, sleep for a little bit, then get the hostname.
+Whatever each job prints to stdout is saved in +task+.+temp+ (under the current working directory); running
+this clusterfile will create 4 files in <code>./fragments/</code>: host0.[hostname], host1.[hostname], host2.[hostname], and host3.[hostname] (where [hostname] is the name of the machine on which the job was run). 
++task+.+username+ and +task+.+password+ are the SSH credentials used to log into the maching -- currently, Clusterfuck
+can only use one global set of credentials. There's no technical reason for this, other than the fact that I don't
+really need to use machine-specific logins, so it'll probably appear in future releases.
++task+.+verbose+ turns on verbose output (messages to stdout each time a job is started, skipped, or canceled).
+
+Once you have a clusterfile you can kick off your jobs by running the command +clusterfuck+ in the same directory.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Add something cool or fix a nefarious bug. Documentation wins extra love.
+* Add tests for it. I'd really like this, but since I haven't written any tests myself yet I can't really blame you if you skip it...
+* Commit, but do not mess with rakefile, version, or history.
+  (if you want to have your own version that's ok -- but
+  bump the version in a separate commit that I can ignore when I pull)
+* Send me a pull request. 
+
+== Copyright
+
+Copyright (c) 2009 Trevor Fountain. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "clusterfuck"
+    gem.summary = %Q{Run jobs across multiple machines via ssh}
+    gem.description = %Q{Automate the execution of jobs across multiple machines with SSH. Ideal for systems with shared filesystems.}
+    gem.email = "doches@gmail.com"
+    gem.homepage = "http://github.com/doches/clusterfuck"
+    gem.authors = ["Trevor Fountain"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+gem 'rdoc'
+require 'rdoc'
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "Clusterfuck #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/*.rb')
+  rdoc.main = "README.rdoc"
+  rdoc.options += ["-SHN","-f","darkfish"]
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/clusterfuck

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'clusterfuck'
+if ARGV[0]
+  # Use specified file
+  load ARGV[0]
+else
+  # Search the current directory for a clusterfile
+  found = false
+  Dir.foreach(".") do |file|
+    if file.downcase == "clusterfile"
+      load file
+      found = true
+      break
+    end
+  end
+  if not found
+    STDERR.puts "No clusterfile found!"
+  end
+end

data/lib/clusterfuck.rb

@@ -0,0 +1,218 @@
+require 'socket'
+require 'net/ssh'
+
+# Clusterfuck is an ugly, dirty hack to run a large number of jobs on multiple machines.
+# If you can break your task up into a series of small, independent jobs, clusterfuck
+# can automate the process of distributing jobs across machines.
+module Clusterfuck
+  # Print a message when a job is cancelled due to too many failures
+  VERBOSE_CANCEL = 0
+  # Print a message when a job is cancelled AND at each failure
+  VERBOSE_FAIL = 1
+  # Print a message for cancellations and failures, AND each time a job is started.
+  VERBOSE_ALL = 2
+
+  # The flag used to prefix dry run (debugging) messages.
+  DEBUG_WARN = "[DRY-RUN]"
+  # The interval to sleep instead of running jobs when performing a dry run (in seconds)
+  DEBUG_INTERVAL = [0.2,1.0]
+
+  # A configuration holds the various pieces of information Clusterfuck needs
+  # to represent a task. 
+  #
+  # You probably won't need to instantiate a Configuration directly; one is created
+  # when you create a new Task, and passed to the block it takes as a parameter. See Task
+  # for more information.
+  #
+  # Possible configuration options include:
+  # [timeout]     Number of seconds to wait before an SSH connection 'times out' (DEFAULT: 2)
+  # [max_fail]    Max number of times a failing job will be re-attempted on a new machine (DEFAULT: 3)
+  # [hosts]       Array of hostnames (or ip addresses) as Strings to use as nodes
+  # [jobs]        Array of Job objects, one per job, which will be allocated to the +hosts+. If you're lazy,
+  #               you can also just use an array of strings (where each string is the command to run) -- a short
+  #               name for each will be produced using the first 8 chars from the command.
+  # [verbose]     Level of message reporting. One of +VERBOSE_CANCEL+,+VERBOSE_FAIL+, or +VERBOSE_ALL+
+  #               (DEFAULT: +VERBOSE_CANCEL+)
+  # [username]    The SSH username to use to connect
+  # [password]    The SSH password to use to connect
+  # [show_report] Show a report after all jobs are complete that gives statistics for each machine.
+  # [debug]       Do a 'dry run' -- allocate jobs to machines and display the result but DO NOT actually
+  #               connect to any machines or run any jobs. Useful for testing your clusterfile before 
+  #               kicking off a major run.
+  # [temp]        Directory in which to capture stdout from each job. Setting this to +false+
+  #               will cause clusterfuck to ignore job output, leaving it up to you to capture the results
+  #               of each job. (DEFAULT: ./fragments)
+  class Configuration
+    # Holds the user-specified options. Again, you probably don't want to access this directly -- use the
+    # getter/setter syntax instead.
+    attr_reader :options
+    
+    # Create a new Configuration object with default options.
+    def initialize
+      @options = {
+          "timeout" => 2,
+          "max_fail" => 3,
+          "verbose" => VERBOSE_CANCEL,
+          "show_report" => true,
+          "temp" => "./fragments",
+        }
+    end
+    
+    # You can get/set options as if they were attributes, i.e. +config.foo = "bar"+ will set the option +foo+ to "bar".
+    def method_missing(key,args=nil)
+      if args.nil?
+        return @options[key.to_s]
+      else
+        key = key.to_s.gsub!("=","")
+        @options[key] = args
+      end
+    end
+    
+    # Get a pretty-printed version of the currently set options
+    def to_s
+      @options.map { |pair| "#{pair[0]} = \"#{pair[1]}\""}.join(", ")
+    end
+    
+    # Convert array of string commands to Job objects if necessary
+    def jobify!
+      @options["jobs"].map! do |job|
+        if not job.is_a?(Job) # Ah-ha, make this string into a job
+          short = job.downcase.gsub(/[^a-z]/,"")
+          short = job[0..7] if short.size > 8 
+          Job.new(short,job)
+        else # Don't change anything...
+          job
+        end
+      end
+    end
+  end
+  
+  # The primary means of interacting with Clusterfuck. Create a new 
+  # Task, passing in a block that takes a Configuration object as a parameter (rake-style).
+  # The constructor returns after all jobs have been completed.
+  class Task
+    # See Configuration for a list of recognized configuration options.
+    def initialize(&custom)
+      # Run configuration options specified in clusterfile
+      config = Configuration.new
+      custom.call(config)
+      config.jobify!
+      
+      # Make output fragment directory
+      `mkdir #{config.temp}` if config.temp and not File.exists?(config.temp)
+      
+      # Run all jobs
+      machines = config.hosts.map { |name| Machine.new(name,config) }
+      machines.each { |machine| machine.run }
+      
+      # Wait for jobs to terminate
+      machines.each do |machine| 
+        begin
+          machine.thread.join
+        rescue Timeout::Error
+          STDERR.puts machine.to_s
+        end
+      end
+      
+      # Print a report, if requested
+      if config.show_report
+        puts " Machine\t| STARTED\t| COMPLETE\t| FAILED\t|"
+        machines.each { |machine| puts machine.report }
+      end
+    end
+  end
+  
+  # Represents a single machine (node) in our ad hoc cluster
+  class Machine
+    # The hostname of this machine
+    attr_accessor :host
+    # The global config options specified when the task was created
+    attr_accessor :config
+    # The thread represented this machine's ssh process
+    attr_reader :thread
+    # The number of jobs this machine has completed
+    attr_reader :jobs_completed
+    # The number of jobs this machine has attempted
+    attr_reader :jobs_attempted
+    # Was this machine dropped from the host list (too many failed jobs)?
+    attr_reader :dropped
+
+    # Create a new machine with the specified +host+ and +config+
+    def initialize(host,config)
+      self.host = host
+      self.config = config
+      
+      @thread = nil
+      @jobs_completed = 0
+      @jobs_attempted = 0
+      @dropped = false
+    end
+    
+    # Open an SSH connection to this machine and process jobs until the global jobs queue is empty
+    def run
+      @thread = Thread.new do
+        while config.jobs.size > 0
+          job = config.jobs.shift
+          if config.debug
+            puts "#{DEBUG_WARN} #{self.host} starting job '#{job.short_name}'"
+            puts "#{DEBUG_WARN}     #{job.command}"
+            delay = rand*(DEBUG_INTERVAL[1]-DEBUG_INTERVAL[0])+DEBUG_INTERVAL[0]
+            @jobs_attempted += 1
+            sleep(delay)
+            @jobs_completed += 1
+          else
+            begin
+              @jobs_attempted += 1
+              Net::SSH.start(self.host,config.username,:password => config.password,:timeout => config.timeout) do |ssh|
+                puts "Starting job #{job.short_name} on #{self.host}" if config.verbose >= VERBOSE_ALL
+                if config.temp
+                  ssh.exec(job.command + " > #{Dir.getwd}/#{config.temp}/#{job.short_name}.#{self.host}")
+                else
+                  ssh.exec(job.command)
+                end
+                @jobs_completed += 1
+              end
+            rescue Timeout::Error
+              puts "#{job.short_name} FAILED on #{self.host}, dropping it from the hostlist" if config.verbose >= VERBOSE_FAIL
+              if not job.failed < config.max_fail
+                config.jobs.push job
+                job.failed += 1
+              else
+                puts "CANCELLING #{job.short_name}, too many failures (#{job.failed})" if config.verbose >= VERBOSE_CANCEL
+              end
+              @dropped = true
+              break
+            end
+          end
+        end
+      end
+    end
+    
+    # Get a one-line summary of this machine's performance
+    def report
+      tab = "\t"
+      if self.host.size > 7
+        tab = ""
+      end
+      "#{self.host}#{tab}\t| #{@jobs_attempted}\t\t| #{@jobs_completed}\t\t| #{@dropped ? 'YES' : 'no'}\t\t|"
+    end
+  end  
+  
+  # Represents an individual job to be run
+  class Job
+    # The short name of this job, used to name the temporary file it produces
+    attr_accessor :short_name
+    # The actual command to run to execute this job.
+    attr_accessor :command
+    # The number of times this job has been unsuccessfully attempted.
+    attr_accessor :failed
+    
+    # Create a new job with the specified short and command
+    def initialize(short_name,command)
+      self.short_name = short_name
+      self.command = command
+      
+      self.failed = 0
+    end    
+  end
+end

data/test/clusterfuck_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class ClusterfuckTest < Test::Unit::TestCase
+  def test_something_for_real
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'clusterfuck'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification 
+name: clusterfuck
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Trevor Fountain
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-20 00:00:00 +01:00
+default_executable: clusterfuck
+dependencies: []
+
+description: Automate the execution of jobs across multiple machines with SSH. Ideal for systems with shared filesystems.
+email: doches@gmail.com
+executables: 
+- clusterfuck
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/clusterfuck
+- lib/clusterfuck.rb
+- test/clusterfuck_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/doches/clusterfuck
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Run jobs across multiple machines via ssh
+test_files: 
+- test/test_helper.rb
+- test/clusterfuck_test.rb