specjour 0.2.5 → 0.3.0.rc1

data/lib/specjour/dispatcher.rb

@@ -2,39 +2,65 @@ module Specjour
   class Dispatcher
     require 'dnssd'
     Thread.abort_on_exception = true
-    include SocketHelpers
+    include SocketHelper
 
     class << self
       attr_accessor :interrupted
       alias interrupted? interrupted
     end
 
-    attr_reader :project_path, :managers, :manager_threads, :hosts
-    attr_accessor :worker_size
+    attr_reader :project_alias, :managers, :manager_threads, :hosts, :options, :all_tests
+    attr_accessor :worker_size, :project_path
 
-    def initialize(project_path)
-      @project_path = project_path
-      @managers = []
+    def initialize(options = {})
+      @options = options
+      @project_path = File.expand_path options[:project_path]
       @worker_size = 0
-      reset_manager_threads
+      @managers = []
+      find_tests
+      clear_manager_threads
     end
 
     def start
-      rsync_daemon.start
+      abort("#{project_path} doesn't exist") unless File.exists?(project_path)
       gather_managers
+      rsync_daemon.start
       dispatch_work
-      printer.join
+      Signal.trap('INT') { Dispatcher.interrupted = true; exit 1 }
+      printer.join if dispatching_tests?
+      wait_on_managers
       exit printer.exit_status
     end
 
     protected
 
-    def all_specs
-      @all_specs ||= Dir.chdir(project_path) do
-        Dir["spec/**/**/*_spec.rb"].sort
+    def find_tests
+      if project_path.match(/(.+)\/((spec|features)(?:\/\w+)*)$/)
+        self.project_path = $1
+        @all_tests = $3 == 'spec' ? all_specs($2) : all_features($2)
+      else
+        @all_tests = Array(all_specs) | Array(all_features)
       end
     end
 
+    def all_specs(tests_path = 'spec')
+      Dir.chdir(project_path) do
+        Dir[File.join(tests_path, "**/*_spec.rb")].sort
+      end if File.exists? File.join(project_path, tests_path)
+    end
+
+    def all_features(tests_path = 'features')
+      Dir.chdir(project_path) do
+        Dir[File.join(tests_path, "**/*.feature")].sort
+      end if File.exists? File.join(project_path, tests_path)
+    end
+
+    def add_manager(manager)
+      set_up_manager(manager)
+      managers << manager
+      self.worker_size += manager.worker_size
+    end
+
     def command_managers(async = false, &block)
       managers.each do |manager|
         manager_threads << Thread.new(manager, &block)
@@ -42,53 +68,83 @@ module Specjour
       wait_on_managers unless async
     end
 
+    def dispatcher_uri
+      @dispatcher_uri ||= URI::Generic.build :scheme => "specjour", :host => hostname, :port => printer.port
+    end
+
     def dispatch_work
+      puts "Managers found: #{managers.size}"
+      puts "Workers found: #{worker_size}"
+      printer.worker_size = worker_size
       command_managers(true) { |m| m.dispatch }
     end
 
+    def dispatching_tests?
+      worker_task == 'run_tests'
+    end
+
     def fetch_manager(uri)
       Timeout.timeout(8) do
         manager = DRbObject.new_with_uri(uri.to_s)
-        if !managers.include?(manager) && manager.available_for?(project_name)
-          set_up_manager(manager, uri)
-          managers << manager
-          self.worker_size += manager.worker_size
+        if !managers.include?(manager) && manager.available_for?(project_alias)
+          add_manager(manager)
         end
       end
     rescue Timeout::Error
       Specjour.logger.debug "Couldn't work with manager at #{uri}"
+    rescue DRb::DRbConnError
+      retry
+    end
+
+    def fork_local_manager
+      puts "No remote managers found, starting a local manager..."
+      manager_options = {:worker_size => options[:worker_size], :registered_projects => [project_alias]}
+      manager = Manager.start_quietly manager_options
+      fetch_manager(manager.drb_uri)
+      at_exit { Process.kill('TERM', manager.pid) rescue Errno::ESRCH }
     end
 
     def gather_managers
-      puts "Waiting for managers"
-      Signal.trap('INT') { self.class.interrupted = true; exit }
+      puts "Looking for managers..."
+      gather_remote_managers
+      fork_local_manager if local_manager_needed?
+      abort "No managers found" if managers.size.zero?
+    end
+
+    def gather_remote_managers
       browser = DNSSD::Service.new
-      begin
-        Timeout.timeout(10) do
-          browser.browse '_druby._tcp' do |reply|
-            if reply.flags.add?
-              resolve_reply(reply)
-            end
-            browser.stop unless reply.flags.more_coming?
+      Timeout.timeout(10) do
+        browser.browse '_druby._tcp' do |reply|
+          if reply.flags.add?
+            resolve_reply(reply)
           end
+          browser.stop unless reply.flags.more_coming?
         end
-      rescue Timeout::Error
       end
-      puts "Managers found: #{managers.size}"
-      abort unless managers.size > 0
-      puts "Workers found: #{worker_size}"
-      printer.worker_size = worker_size
+      rescue Timeout::Error
+    end
+
+    def local_manager_needed?
+      options[:worker_size] > 0 && no_local_managers?
+    end
+
+    def no_local_managers?
+      !managers.any? {|m| m.hostname == hostname}
     end
 
     def printer
-      @printer ||= Printer.start(all_specs)
+      @printer ||= Printer.start(all_tests)
+    end
+
+    def project_alias
+      @project_alias ||= options[:project_alias] || project_name
     end
 
     def project_name
       @project_name ||= File.basename(project_path)
     end
 
-    def reset_manager_threads
+    def clear_manager_threads
       @manager_threads = []
     end
 
@@ -105,15 +161,22 @@ module Specjour
       @rsync_daemon ||= RsyncDaemon.new(project_path, project_name)
     end
 
-    def set_up_manager(manager, uri)
+    def set_up_manager(manager)
       manager.project_name = project_name
-      manager.dispatcher_uri = URI::Generic.build :scheme => "specjour", :host => hostname, :port => printer.port
-      at_exit { manager.kill_worker_processes }
+      manager.dispatcher_uri = dispatcher_uri
+      manager.preload_spec = all_tests.detect {|f| f =~ /_spec\.rb$/}
+      manager.preload_feature = all_tests.detect {|f| f =~ /\.feature$/}
+      manager.worker_task = worker_task
+      at_exit { manager.kill_worker_processes rescue DRb::DRbConnError }
     end
 
     def wait_on_managers
       manager_threads.each {|t| t.join; t.exit}
-      reset_manager_threads
+      clear_manager_threads
+    end
+
+    def worker_task
+      options[:worker_task] || 'run_tests'
     end
   end
 end

data/lib/specjour/manager.rb

@@ -1,29 +1,33 @@
 module Specjour
   class Manager
     require 'dnssd'
+    require 'specjour/rspec'
+    require 'specjour/cucumber'
+
     include DRbUndumped
-    include SocketHelpers
+    include SocketHelper
+
+    attr_accessor :project_name, :preload_spec, :preload_feature, :worker_task, :pid
+    attr_reader :worker_size, :dispatcher_uri, :registered_projects, :bonjour_service, :worker_pids, :options
 
-    attr_accessor :project_name, :specs_to_run
-    attr_reader :worker_size, :batch_size, :dispatcher_uri, :registered_projects, :bonjour_service, :worker_pids
+    def self.start_quietly(options)
+      manager = new options.merge(:quiet => true)
+      manager.drb_uri
+      manager.pid = QuietFork.fork { manager.start }
+      manager
+    end
 
     def initialize(options = {})
+      @options = options
       @worker_size = options[:worker_size]
-      @batch_size = options[:batch_size]
+      @worker_task = options[:worker_task]
       @registered_projects = options[:registered_projects]
       @worker_pids = []
+      at_exit { kill_worker_processes }
     end
 
     def available_for?(project_name)
-      registered_projects ? registered_projects.include?(project_name) : true
-   end
-
-    def bundle_install
-      Dir.chdir(project_path) do
-        unless system('bundle check > /dev/null')
-          system("bundle install --relock > /dev/null")
-        end
-      end
+      registered_projects ? registered_projects.include?(project_name) : false
     end
 
     def dispatcher_uri=(uri)
@@ -31,50 +35,67 @@ module Specjour
       @dispatcher_uri = uri
     end
 
-    def kill_worker_processes
-      Process.kill('TERM', *worker_pids) rescue nil
-    end
-
-    def project_path
-      File.join("/tmp", project_name)
-    end
-
     def dispatch
       suspend_bonjour do
         sync
-        bundle_install
+        in_project { Configuration.before_fork.call }
         dispatch_workers
       end
     end
 
+    def drb_start
+      DRb.start_service drb_uri.to_s, self
+      at_exit { DRb.stop_service }
+    end
+
+    def drb_uri
+      @drb_uri ||= begin
+        current_uri.scheme = "druby"
+        current_uri
+      end
+    end
+
     def dispatch_workers
-      GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
+      worker_pids.clear
       (1..worker_size).each do |index|
         worker_pids << fork do
-          exec("specjour --batch-size #{batch_size} #{'--log' if Specjour.log?} --do-work #{project_path},#{dispatcher_uri},#{index}")
-          Kernel.exit!
+          exec "specjour work #{worker_options(index)}"
         end
       end
-      at_exit { kill_worker_processes }
       Process.waitall
     end
 
+    def in_project(&block)
+      Dir.chdir(project_path, &block)
+    end
+
+    def kill_worker_processes
+      Process.kill('TERM', *worker_pids) rescue Errno::ESRCH
+    end
+
+    def pid
+      @pid || Process.pid
+    end
+
+    def project_path
+      File.join("/tmp", project_name)
+    end
+
     def start
       drb_start
-      puts "Workers ready: #{worker_size}"
+      puts "Workers ready: #{worker_size}."
+      puts "Listening for #{registered_projects.join(', ')}"
       bonjour_announce
-      Signal.trap('INT') { puts; puts "Shutting down manager..."; exit }
+      Signal.trap('INT') { puts; puts "Shutting down manager..."; exit 1 }
       DRb.thread.join
     end
 
-    def drb_start
-      DRb.start_service nil, self
-      puts "Manager started at #{drb_uri}"
-      at_exit { DRb.stop_service }
+    def quiet?
+      options.has_key? :quiet
     end
 
     def sync
-      cmd "rsync -a --delete --port=8989 #{dispatcher_uri.host}::#{project_name} #{project_path}"
+      cmd "rsync -aL --delete --port=8989 #{dispatcher_uri.host}::#{project_name} #{project_path}"
     end
 
     protected
@@ -88,14 +109,19 @@ module Specjour
       system command
     end
 
-    def drb_uri
-      @drb_uri ||= URI.parse(DRb.uri)
-    end
-
     def suspend_bonjour(&block)
       bonjour_service.stop
       block.call
       bonjour_announce
     end
+
+    def worker_options(index)
+      exec_options = "--project-path #{project_path} --printer-uri #{dispatcher_uri} --number #{index} --task #{worker_task}"
+      exec_options << " --preload-spec #{preload_spec}" if preload_spec
+      exec_options << " --preload-feature #{preload_feature}" if preload_feature
+      exec_options << " --log" if Specjour.log?
+      exec_options << " --quiet" if quiet?
+      exec_options
+    end
   end
 end

data/lib/specjour/printer.rb

@@ -1,5 +1,6 @@
 module Specjour
   require 'specjour/rspec'
+  require 'specjour/cucumber'
 
   class Printer < GServer
     include Protocol
@@ -44,11 +45,15 @@ module Specjour
     end
 
     def exit_status
-      report.exit_status
+      reporters.all? {|r| r.exit_status == true}
     end
 
-    def worker_summary=(client, summary)
-      report.add(summary)
+    def rspec_summary=(client, summary)
+      rspec_report.add(summary)
+    end
+
+    def cucumber_summary=(client, summary)
+      cucumber_report.add(summary)
     end
 
     protected
@@ -66,7 +71,7 @@ module Specjour
     end
 
     def error(exception)
-      Specjour.logger.debug exception.inspect
+      Specjour.logger.debug "#{exception.inspect}\n#{exception.backtrace.join("\n")}"
     end
 
     def process(message, client)
@@ -78,22 +83,39 @@ module Specjour
       end
     end
 
-    def report
-      @report ||= Rspec::FinalReport.new
+    def rspec_report
+      @rspec_report ||= Rspec::FinalReport.new
+    end
+
+    def cucumber_report
+      @cucumber_report ||= Cucumber::FinalReport.new
+    end
+
+    def reporters
+      [@rspec_report, @cucumber_report].compact
     end
 
     def stopping
-      report.summarize
-      if disconnections != completed_workers && !Specjour::Dispatcher.interrupted?
-        puts abandoned_worker_message
-      end
+      summarize_reports
+      warn_if_workers_deserted
+    end
+
+    def summarize_reports
+      reporters.each {|r| r.summarize}
     end
 
     def synchronize(&block)
       @connectionsMutex.synchronize &block
     end
 
-    def abandoned_worker_message
+    def warn_if_workers_deserted
+      if disconnections != completed_workers && !Dispatcher.interrupted?
+        puts
+        puts workers_deserted_message
+      end
+    end
+
+    def workers_deserted_message
       data = "* ERROR: NOT ALL WORKERS COMPLETED PROPERLY *"
       filler = "*" * data.size
       [filler, data, filler].join "\n"

data/lib/specjour/quiet_fork.rb

@@ -0,0 +1,11 @@
+module Specjour::QuietFork
+  extend self
+  attr_reader :pid
+
+  def self.fork(&block)
+    @pid = Kernel.fork do
+      $stdout = StringIO.new
+      block.call
+    end
+  end
+end

data/lib/specjour/rspec/distributed_formatter.rb

@@ -2,11 +2,6 @@ module Specjour::Rspec
   class DistributedFormatter < Spec::Runner::Formatter::BaseTextFormatter
     require 'specjour/rspec/marshalable_rspec_failure'
 
-    class << self
-      attr_accessor :batch_size
-    end
-    @batch_size = 1
-
     attr_reader :failing_messages, :passing_messages, :pending_messages, :output
     attr_reader :duration, :example_count, :failure_count, :pending_count, :pending_examples, :failing_examples
 
@@ -22,18 +17,18 @@ module Specjour::Rspec
 
     def example_failed(example, counter, failure)
       failing_messages << colorize_failure('F', failure)
-      batch_print(failing_messages)
+      print_and_flush(failing_messages)
     end
 
     def example_passed(example)
       passing_messages << green('.')
-      batch_print(passing_messages)
+      print_and_flush(passing_messages)
     end
 
     def example_pending(example, message, deprecated_pending_location=nil)
       super
       pending_messages << yellow('*')
-      batch_print(pending_messages)
+      print_and_flush(pending_messages)
     end
 
     def dump_summary(duration, example_count, failure_count, pending_count)
@@ -41,7 +36,7 @@ module Specjour::Rspec
       @example_count = example_count
       @failure_count = failure_count
       @pending_count = pending_count
-      output.send_message(:worker_summary=, to_hash)
+      output.send_message(:rspec_summary=, to_hash)
     end
 
     def dump_pending
@@ -68,12 +63,6 @@ module Specjour::Rspec
 
     protected
 
-    def batch_print(messages)
-      if messages.size == self.class.batch_size
-        print_and_flush(messages)
-      end
-    end
-
     def print_and_flush(messages)
       output.print messages.to_s
       output.flush

data/lib/specjour/rspec/preloader.rb

@@ -0,0 +1,8 @@
+class Specjour::Rspec::Preloader
+  def self.load(spec_file)
+    $LOAD_PATH.unshift File.join(Dir.pwd, 'spec')
+    require spec_file
+  ensure
+    $LOAD_PATH.shift
+  end
+end

data/lib/specjour/rspec.rb

@@ -5,5 +5,6 @@ module Specjour
 
     require 'specjour/rspec/distributed_formatter'
     require 'specjour/rspec/final_report'
+    require 'specjour/rspec/preloader'
   end
 end

data/lib/specjour/rsync_daemon.rb

@@ -1,7 +1,7 @@
 module Specjour
   class RsyncDaemon
     require 'fileutils'
-    include SocketHelpers
+    include SocketHelper
 
     # Corresponds to the version of specjour that changed the configuration
     # file.

data/lib/specjour/socket_helper.rb

@@ -0,0 +1,28 @@
+module Specjour
+  module SocketHelper
+    def ip_from_hostname(hostname)
+      Socket.getaddrinfo(hostname, nil, Socket::AF_INET, Socket::SOCK_STREAM).first.fetch(3)
+    end
+
+    def hostname
+      @hostname ||= Socket.gethostname
+    end
+
+    def current_uri
+      @current_uri ||= new_uri
+    end
+
+    def new_uri
+      URI::Generic.build :host => faux_server[2], :port => faux_server[1]
+    end
+
+    protected
+
+    def faux_server
+      server = TCPServer.new('0.0.0.0', nil)
+      server.addr
+    ensure
+      server.close
+    end
+  end
+end

data/lib/specjour/worker.rb

@@ -1,51 +1,78 @@
 module Specjour
+  require 'specjour/rspec'
   require 'specjour/cucumber'
 
   class Worker
     include Protocol
-    include SocketHelpers
+    include SocketHelper
     attr_accessor :printer_uri
-    attr_reader :project_path, :specs_to_run, :number, :batch_size
-
-    def initialize(project_path, printer_uri, number, batch_size)
-      @project_path = project_path
-      @specs_to_run = specs_to_run
-      @number = number.to_i
-      @batch_size = batch_size.to_i
-      self.printer_uri = printer_uri
-      Rspec::DistributedFormatter.batch_size = batch_size
+    attr_reader :project_path, :number, :preload_spec, :preload_feature, :task
+
+    def initialize(options = {})
+      ARGV.replace []
+      $stdout = StringIO.new if options[:quiet]
+      @project_path = options[:project_path]
+      @number = options[:number].to_i
+      @preload_spec = options[:preload_spec]
+      @preload_feature = options[:preload_feature]
+      @task = options[:task]
+      self.printer_uri = options[:printer_uri]
       set_env_variables
+      Dir.chdir(project_path)
     end
 
     def printer_uri=(val)
       @printer_uri = URI.parse(val)
     end
 
-    def run
+    def prepare
+      load_app
+      Configuration.prepare.call
+    end
+
+    def run_tests
+      load_app
+      Configuration.after_fork.call
       run_time = 0
-      Dir.chdir(project_path)
+
       while test = connection.next_test
-        run_time += Benchmark.realtime do
+        time = Benchmark.realtime do
           run_test test
         end
+        run_time += time if test =~ /_spec\.rb$/
       end
-      connection.send_message(:worker_summary=, {:duration => sprintf("%6f", run_time)})
+      connection.send_message(:rspec_summary=, {:duration => sprintf("%6f", run_time)})
       connection.send_message(:done)
       connection.disconnect
     end
 
+    def start
+      send task
+    end
+
     protected
 
     def connection
       @connection ||= printer_connection
     end
 
+    def load_app
+      Rspec::Preloader.load(preload_spec) if preload_spec
+      Cucumber::Preloader.load(preload_feature) if preload_feature
+    end
+
     def printer_connection
       Connection.new printer_uri
     end
 
+    def print_status(test)
+      status = "[#{ENV['TEST_ENV_NUMBER']}] Running #{test}"
+      puts status
+      $PROGRAM_NAME = "specjour#{status}"
+    end
+
     def run_test(test)
-      puts "[#{ENV['TEST_ENV_NUMBER']}] Running #{test}"
+      print_status(test)
       if test =~ /\.feature$/
         run_feature test
       else
@@ -54,7 +81,6 @@ module Specjour
     end
 
     def run_feature(feature)
-      set_up_cucumber
       cli = ::Cucumber::Cli::Main.new(['--format', 'Specjour::Cucumber::DistributedFormatter', feature], connection)
       cli.execute!(::Cucumber::Cli::Main.step_mother)
     end
@@ -70,17 +96,8 @@ module Specjour
     end
 
     def set_env_variables
-      ENV['PREPARE_DB'] = 'true'
       ENV['RSPEC_COLOR'] = 'true'
       ENV['TEST_ENV_NUMBER'] = number.to_s
     end
-
-    def set_up_cucumber
-      unless @cucumber_loaded
-        Cucumber::DistributedFormatter.batch_size = batch_size
-        ::Cucumber::Cli::Options.class_eval { def print_profile_information; end }
-        @cucumber_loaded = true
-      end
-    end
   end
 end