minimal-buffet 0.6

data/bin/buffet

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'buffet/cli'
+
+Buffet::CLI.new ARGV

data/lib/buffet/cli.rb

@@ -0,0 +1,29 @@
+require 'buffet'
+require 'optparse'
+
+module Buffet
+  class CLI
+    def initialize args
+      opts = OptionParser.new do |opts|
+        opts.banner = "Usage: buffet [options] [spec-files]"
+
+        opts.on('-c', '--config CONFIG',
+                'Use the specified CONFIG file') do |config_file|
+          Settings.load_file File.expand_path(config_file)
+        end
+
+        opts.on('-p', '--project PROJECT',
+                'Use the specified PROJECT name') do |project_name|
+          Settings.project_name = project_name
+        end
+      end.parse!(args)
+
+      specs = Buffet.extract_specs_from(opts.empty? ? 'spec' : opts)
+
+      runner = Runner.new
+      runner.run specs
+
+      exit 1 if runner.failures?
+    end
+  end
+end

data/lib/buffet/command_runner.rb

@@ -0,0 +1,23 @@
+require 'logger'
+require 'wopen3'
+
+module Buffet
+  class CommandRunner
+    def initialize logger = Logger.new(STDOUT)
+      @logger = logger
+    end
+
+    def run *command
+      start_time = Time.now
+      result = Wopen3.system *command
+      end_time = Time.now
+      @logger.info "\n" +
+        "command: #{command.join ' '}\n" +
+        "time: #{end_time - start_time}\n" +
+        "status: #{result.status}\n" +
+        "stdout:\n#{result.stdout}\n" +
+        "stderr:\n#{result.stderr}"
+      result
+    end
+  end
+end

data/lib/buffet/master.rb

@@ -0,0 +1,129 @@
+require 'benchmark'
+require 'drb'
+require 'thread'
+require 'socket'
+
+module Buffet
+  class Master
+    attr_reader :failures, :stats
+
+    def initialize project, slaves, specs, listener
+      @project = project
+      @slaves = slaves
+      @stats = {:examples => 0, :failures => 0, :pending => 0}
+      @slaves_stats = Hash[@slaves.map do |slave|
+        [slave.user_at_host, stats.dup.merge!(:slave => slave)]
+      end]
+      @stats[:slaves] = @slaves_stats
+      @lock = Mutex.new
+      @failures = []
+      @specs = specs.shuffle # Never have the same test distribution
+      @listener = listener
+    end
+
+    def run
+      start_service
+
+      @stats[:total_time] = Benchmark.measure do
+        threads = @slaves.map do |slave|
+          Thread.new do
+            time = Benchmark.measure do
+              prepare_slave slave
+              run_slave slave
+            end.real
+            @lock.synchronize { @slaves_stats[slave.name][:total_time] = time }
+          end
+        end
+
+        threads.each { |t| t.join }
+      end.real
+
+      stop_service
+    end
+
+    def next_file_for slave_name
+      file = @lock.synchronize { @specs.shift }
+      if file
+        slave = @slaves_stats[slave_name][:slave]
+        @listener.spec_taken slave, file if file
+      end
+      file
+    end
+
+    def example_passed slave_name, details
+      @lock.synchronize do
+        @stats[:examples] += 1
+        @slaves_stats[slave_name][:examples] += 1
+      end
+
+      @listener.example_passed
+    end
+
+    def example_failed slave_name, details
+      @lock.synchronize do
+        @stats[:examples] += 1
+        @stats[:failures] += 1
+        @slaves_stats[slave_name][:failures] += 1
+        @failures << details
+      end
+
+      @listener.example_failed
+    end
+
+    def example_pending slave_name, details
+     @lock.synchronize do
+       @stats[:examples] += 1
+       @stats[:pending] += 1
+       @slaves_stats[slave_name][:pending] += 1
+     end
+
+     @listener.example_pending
+    end
+
+    private
+
+    def server_uri
+      @drb_server.uri
+    end
+
+    def start_service
+      @drb_server = DRb.start_service("druby://#{ip}:0", self)
+    end
+
+    def stop_service
+      DRb.stop_service
+    end
+
+    def ip
+      result = Buffet.run! 'host `hostname -s`'
+      result.stdout.chomp.match(/((\d+\.){3}\d+)/)[1]
+    end
+
+    def prepare_slave slave
+      time = Benchmark.measure do
+        @project.sync_to slave
+
+        if Settings.has_prepare_script?
+          slave.execute_in_project "#{Settings.prepare_script} #{Buffet.user} #{@project.name}"
+        end
+
+        # Copy support files so they can be run on the remote machine
+        slave.scp File.dirname(__FILE__) + '/../../support',
+                  @project.support_dir_on_slave, :recurse => true
+      end.real
+      @lock.synchronize { @slaves_stats[slave.name][:prepare_time] = time }
+
+      @listener.slave_prepared slave
+    end
+
+    def run_slave slave
+      time = Benchmark.measure do
+      slave.execute_in_project(
+        ".buffet/buffet-worker #{server_uri} #{slave.user_at_host} #{Settings.framework}")
+      end.real
+      @lock.synchronize { @slaves_stats[slave.name][:test_time] = time }
+
+      @listener.slave_finished slave
+    end
+  end
+end

data/lib/buffet/project.rb

@@ -0,0 +1,24 @@
+module Buffet
+  class Project
+    attr_accessor :name
+    attr_reader   :directory
+
+    def initialize directory
+      @name = File.basename directory
+      @directory = File.expand_path directory
+    end
+
+    def directory_on_slave
+      "#{Buffet.workspace_dir}/#{name}"
+    end
+
+    def support_dir_on_slave
+      "#{directory_on_slave}/.buffet"
+    end
+
+    def sync_to slave
+      slave.execute "mkdir -p #{directory_on_slave}"
+      slave.rsync directory + '/', directory_on_slave
+    end
+  end
+end

data/lib/buffet/runner.rb

@@ -0,0 +1,93 @@
+require 'buffet'
+require 'colorize'
+
+module Buffet
+  class Runner
+    def initialize
+      @project = Settings.project
+    end
+
+    def run specs = nil
+      @specs = specs
+      raise 'No specs found' if @specs.empty?
+
+      @slaves = Settings.slaves
+      raise 'No slaves defined in settings.yml' if @slaves.empty?
+
+      Buffet.logger.info "Starting Buffet test run"
+      puts "Running Buffet..."
+
+      run_tests
+      display_results
+    end
+
+    def slave_prepared slave
+      Buffet.logger.info "#{slave.name} prepared"
+    end
+
+    def spec_taken slave, spec_file
+      Buffet.logger.info "#{slave.name} took #{spec_file}"
+    end
+
+    def example_passed
+      print '.'.green
+      STDOUT.flush
+    end
+
+    def example_failed
+      print 'F'.red
+      STDOUT.flush
+    end
+
+    def example_pending
+      print '*'.yellow
+      STDOUT.flush
+    end
+
+    def slave_finished slave
+      Buffet.logger.info "#{slave.name} finished"
+    end
+
+    def failures?
+      @master.failures.any?
+    end
+
+    private
+
+    def run_tests
+      @master = Master.new @project, @slaves, @specs, self
+      @master.run
+    end
+
+    def display_results
+      results = []
+      results << "\n"
+
+      @master.stats[:slaves].each do |slave_name, slave_stats|
+        results << "#{slave_name}:"
+        slave_stats.each do |key, value|
+          results << "\t#{key}: #{value}" unless key == :slave
+        end
+      end
+
+      results << "Total Examples: #{@master.stats[:examples]}"
+      results << "Total Pending:  #{@master.stats[:pending]}"
+      results << "Total Failures: #{@master.stats[:failures]}"
+      results << ''
+      results << "Buffet consumed in #{@master.stats[:total_time]} seconds"
+
+      unless @master.failures.empty?
+        results << ''
+        results << @master.failures.map do |failure|
+          "#{failure[:description]}\n".red +
+          "Slave: #{failure[:slave_name]}\n" +
+          "Location: #{failure[:location]}\n" +
+          "#{failure[:message]}\n" +
+          "#{failure[:backtrace]}\n"
+        end
+      end
+
+      puts results
+    end
+  end
+end

data/lib/buffet/settings.rb

@@ -0,0 +1,58 @@
+require 'yaml'
+
+module Buffet
+  class Settings
+    DEFAULT_SETTINGS_FILE       = 'buffet.yml'
+    DEFAULT_PREPARE_SCRIPT      = 'bin/before-buffet-run'
+    DEFAULT_EXCLUDE_FILTER_FILE = '.buffet-exclude-filter'
+
+    class << self
+      def [](name)
+        @settings ||= load_file DEFAULT_SETTINGS_FILE
+        @settings[name]
+      end
+
+      def load_file file
+        @settings = YAML.load_file file
+      end
+
+      def slaves
+        @slaves ||= self['slaves'].map do |slave_hash|
+          Slave.new slave_hash['user'], slave_hash['host'], project
+        end
+      end
+
+      def project_name=(project_name)
+        project.name = project_name
+      end
+
+      def project
+        @project ||= Project.new Dir.pwd
+      end
+
+      def framework
+        self['framework'].upcase || 'RSPEC1'
+      end
+
+      def prepare_script
+        self['prepare_script'] || DEFAULT_PREPARE_SCRIPT
+      end
+
+      def has_prepare_script?
+        self['prepare_script'] || File.exist?(DEFAULT_PREPARE_SCRIPT)
+      end
+
+      def exclude_filter_file
+        self['exclude_filter_file'] || DEFAULT_EXCLUDE_FILTER_FILE
+      end
+
+      def has_exclude_filter_file?
+        self['exclude_filter_file'] || File.exist?(DEFAULT_EXCLUDE_FILTER_FILE)
+      end
+
+      def reset!
+        @settings = nil
+      end
+    end
+  end
+end

data/lib/buffet/slave.rb

@@ -0,0 +1,48 @@
+module Buffet
+  class Slave
+    attr_reader :user, :host
+
+    def initialize user, host, project
+      @user = user
+      @host = host
+      @project = project
+    end
+
+    def rsync src, dest
+      Buffet.run! 'rsync', '-aqz', '--delete',
+                  '--delete-excluded', rsync_exclude_flags,
+                  '-e', 'ssh', src, "#{user_at_host}:#{dest}"
+    end
+
+    def scp src, dest, options = {}
+      args = [src, "#{user_at_host}:#{dest}"]
+      args.unshift '-r' if options[:recurse]
+      Buffet.run! 'scp', *args
+    end
+
+    def execute_in_project command
+      execute "cd #{@project.directory_on_slave} && #{command}"
+    end
+
+    def execute command
+      Buffet.run! 'ssh', "#{user_at_host}", command
+    end
+
+    def name
+      user_at_host
+    end
+
+    def user_at_host
+      "#{@user}@#{@host}"
+    end
+
+    private
+
+    def rsync_exclude_flags
+      if Settings.has_exclude_filter_file?
+        exclude_flags = "--exclude-from=#{Settings.exclude_filter_file}"
+      end
+      exclude_flags || ''
+    end
+  end
+end

data/lib/buffet/version.rb

@@ -0,0 +1,3 @@
+module Buffet
+  VERSION = '0.6'
+end

data/lib/buffet.rb

@@ -0,0 +1,60 @@
+require 'fileutils'
+require 'find'
+require 'logger'
+require 'pathname'
+
+module Buffet
+  autoload :CommandRunner, 'buffet/command_runner'
+  autoload :Master, 'buffet/master'
+  autoload :Project, 'buffet/project'
+  autoload :Runner, 'buffet/runner'
+  autoload :Settings, 'buffet/settings'
+  autoload :Slave, 'buffet/slave'
+
+  def self.logdir
+    @logdir ||= Pathname.new(ENV['HOME']) + '.buffet/log'
+  end
+
+  def self.logfile
+    'buffet.log'
+  end
+
+  def self.logger
+    @logger ||= begin
+      FileUtils.mkdir_p logdir
+      Logger.new logdir + logfile
+    end
+  end
+
+  def self.runner
+    @runner ||= CommandRunner.new logger
+  end
+
+  def self.run! *command
+    result = runner.run *command
+    unless result.success?
+      logger.error 'exiting due to non-zero exit status'
+      exit result.status
+    end
+    result
+  end
+
+  # Given a set of files/directories, return all spec files contained
+  def self.extract_specs_from files
+    specs = []
+    files.each do |spec_file|
+      Find.find(spec_file) do |f|
+        specs << f if f.match /_spec\.rb$/
+      end
+    end
+    specs.uniq
+  end
+
+  def self.workspace_dir
+    ".buffet/workspaces/#{user}" # Relative to home directory
+  end
+
+  def self.user
+    @user ||= `whoami`.chomp
+  end
+end

data/support/buffet-worker

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'drb'
+require 'fileutils'
+
+if File.exist? 'Gemfile'
+  require 'rubygems'
+  require 'bundler/setup'
+end
+
+# NOTE: ARGV is used by the spec runner. If I leave the drb server address
+# in ARGV, rspec will think it's an argument to the test runner.
+buffet_server = DRbObject.new_with_uri(ARGV.shift)
+slave_name = ARGV.shift
+framework = ARGV.shift
+
+FileUtils.mkdir_p('./tmp')
+
+if framework == 'RSPEC1'
+  require 'spec'
+  require 'spec/runner/command_line'
+  require File.dirname(__FILE__) + '/rspec1_formatter.rb'
+
+  Spec::Runner::Formatter::AugmentedTextFormatter.configure buffet_server, slave_name
+
+  while file = buffet_server.next_file_for(slave_name)
+    # RSpec1 closes stderr/out after each run, so we reopen them each time
+    outlog = File.open('./tmp/buffet.out.log', 'a')
+    errlog = File.open('./tmp/buffet.error.log', 'a')
+
+    Spec::Runner::CommandLine.run(
+      Spec::Runner::OptionParser.parse(
+        ['--format', 'Spec::Runner::Formatter::AugmentedTextFormatter', file],
+        errlog,
+        outlog
+      )
+    )
+  end
+else
+  require 'rspec'
+  require File.dirname(__FILE__) + '/rspec2_formatter.rb'
+
+  RSpec::Core::Formatters::AugmentedTextFormatter.configure buffet_server, slave_name
+
+  while file = buffet_server.next_file_for(slave_name)
+    RSpec::Core::CommandLine.new(
+      ['--format', 'RSpec::Core::Formatters::AugmentedTextFormatter', file]
+    ).run($stderr, $stdout)
+
+    RSpec.world.reset
+  end
+end

data/support/rspec1_formatter.rb

@@ -0,0 +1,44 @@
+require 'spec/runner/formatter/base_text_formatter'
+
+module Spec
+  module Runner
+    module Formatter
+      class AugmentedTextFormatter < BaseTextFormatter
+        def self.configure buffet_server, slave_name
+          @@buffet_server = buffet_server
+          @@slave_name = slave_name
+        end
+
+        def example_passed example_proxy
+          super
+          @@buffet_server.example_passed(@@slave_name, {
+            :description => example_proxy.description,
+            :location    => example_proxy.location,
+            :slave_name  => @@slave_name,
+          })
+        end
+
+        def example_failed example_proxy, counter, failure
+          super
+          @@buffet_server.example_failed(@@slave_name, {
+            :backtrace   => failure.exception.backtrace.join("\n"),
+            :description => failure.header,
+            :location    => example_proxy.location,
+            :message     => failure.exception.message,
+            :slave_name  => @@slave_name,
+          })
+        end
+
+        def example_pending example, message, deprecated_pending_location=nil
+          super
+          @@buffet_server.example_pending(@@slave_name, {
+            :description => example.description,
+            :location    => example.location,
+            :message     => message,
+            :slave_name  => @@slave_name,
+          })
+        end
+      end
+    end
+  end
+end

data/support/rspec2_formatter.rb

@@ -0,0 +1,51 @@
+require 'rspec/core/formatters/base_text_formatter'
+
+module RSpec
+  module Core
+    module Formatters
+      class AugmentedTextFormatter < BaseTextFormatter
+        def initialize(output)
+          super(output)
+        end
+
+        def self.configure buffet_server, slave_name
+          @@buffet_server = buffet_server
+          @@slave_name = slave_name
+        end
+
+        def example_passed example
+          super
+          @@buffet_server.example_passed(@@slave_name, {
+            :description => example.description,
+            :location    => example.location,
+            :slave_name  => slave_name,
+          })
+        end
+
+        def example_failed example
+          super
+          exception = example.metadata[:execution_result][:exception]
+          backtrace = format_backtrace(exception.backtrace, example).join("\n")
+
+          @@buffet_server.example_failed(@@slave_name, {
+            :description => description,
+            :backtrace   => backtrace,
+            :message     => exception.message,
+            :location    => example.location,
+            :slave_name  => slave_name,
+          })
+        end
+
+        def example_pending example, message, deprecated_pending_location=nil
+          super
+          @@buffet_server.example_pending(@@slave_name, {
+            :description => example.description,
+            :location    => example.location,
+            :message     => message,
+            :slave_name  => slave_name,
+          })
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification 
+name: minimal-buffet
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 6
+  version: "0.6"
+platform: ruby
+authors: 
+- Causes Engineering
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-12-29 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: colorize
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: wopen3
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: mkdtemp
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: Buffet distributes RSpec test cases over multiple machines.
+email: 
+- eng@causes.com
+- grant@causes.com
+- shane@causes.com
+executables: 
+- buffet
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/buffet.rb
+- lib/buffet/cli.rb
+- lib/buffet/command_runner.rb
+- lib/buffet/master.rb
+- lib/buffet/project.rb
+- lib/buffet/runner.rb
+- lib/buffet/settings.rb
+- lib/buffet/slave.rb
+- lib/buffet/version.rb
+- support/buffet-worker
+- support/rspec1_formatter.rb
+- support/rspec2_formatter.rb
+has_rdoc: true
+homepage: http://github.com/causes/buffet
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Distributed testing framework for Ruby, Rails and RSpec
+test_files: []
+