deep_test 1.1.4 → 1.2.0

data/lib/deep_test/ui/console.rb

@@ -0,0 +1,76 @@
+module DeepTest
+  module UI
+    class Console
+      def initialize(options)
+      end
+
+      METHOD_DESCRIPTIONS = {
+        :sync => "Synchronizing working copies on worker machines",
+        :spawn_worker_server => "Spawning test environment processes",
+        :load_files => "Loading test files for workers",
+        :start_all => "Starting workers",
+        :stop_all => "Stopping workers"
+      } unless defined?(METHOD_DESCRIPTIONS)
+
+      def distributed_failover_to_local(method, exception)
+        width = 70
+        puts " Distributed DeepTest Failure ".center(width, '*')
+        puts "*   Failed during #{method}".ljust(width - 1) + "*"
+        puts "* #{exception.message}".ljust(width - 1) + "*"
+        puts "* Failing over to local run".ljust(width - 1) + "*"
+        puts "*" * width
+      end
+
+      def dispatch_starting(method_name)
+        @spinner.stop if @spinner
+        @spinner = Spinner.new(label(method_name))
+        @spinner.start
+      end
+
+      def label(method_name)
+        METHOD_DESCRIPTIONS[method_name.to_sym] || method_name.to_s
+      end
+
+      def dispatch_finished(method_name)
+        @spinner.stop if @spinner
+        @spinner = nil
+      end
+
+      class Spinner
+        FRAMES = ['|', '/', '-', '\\'] unless defined?(FRAMES)
+        BACKSPACE = "\x08"  unless defined?(BACKSPACE)
+        SECONDS_PER_FRAME = 0.5 / 4 unless defined?(SECONDS_PER_FRAME)
+
+        def initialize(label)
+          @label = label
+        end
+
+        def start
+          @start_time = Time.now
+          show "#{@label}: "
+          @thread = Thread.new do
+            index = 0
+            loop do
+              show FRAMES[index]
+              sleep SECONDS_PER_FRAME
+              show BACKSPACE
+              index = (index + 1) % FRAMES.length
+            end
+          end 
+        end
+
+        def stop
+          @stop_time = Time.now
+          @thread.kill if @thread
+          show BACKSPACE
+          show("finished in %.2f seconds\n" % (@stop_time.to_f - @start_time.to_f))
+        end
+
+        def show(string)
+          $stdout.print string
+          $stdout.flush
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/ui/null.rb

@@ -0,0 +1,17 @@
+module DeepTest
+  module UI
+    class Null
+      def initialize(options)
+      end
+
+      def distributed_failover_to_local(method, exception)
+      end
+
+      def dispatch_starting(method_name)
+      end
+
+      def dispatch_finished(method_name)
+      end
+    end
+  end
+end

data/lib/deep_test/warlock.rb

@@ -1,40 +1,91 @@
 module DeepTest
   class Warlock
     def initialize
+      @demons_semaphore = Mutex.new
       @demons = []
+      @reapers = []
     end
-    
+
     def start(name, &block)
+      # Not synchronizing for the fork seems to cause
+      # random errors (Bus Error, Segfault, and GC non-object)
+      # in RemoteWorkerServer processes.
+      #
       begin
-        pid = Process.fork do
-          Signal.trap("HUP") { exit 0 }
-          yield
-          exit
+        pid = nil
+        @demons_semaphore.synchronize do 
+          pid = DeepTest.drb_safe_fork do
+            # Fork leaves the semaphore locked and we'll never make it
+            # to end of synchronize block.
+            #
+            # The Ruby 1.8.6 C mutex implementation automatically treats
+            # a mutex locked by a dead thread as unlocked and will raise
+            # an error if we try to unlock it from this thread.
+            #
+            @demons_semaphore.unlock if @demons_semaphore.locked?
+
+            begin
+              yield
+            rescue Exception => e
+              DeepTest.logger.debug "Exception in #{name} (#{Process.pid}): #{e.message}"
+              raise
+            end
+
+            exit
+          end
+
+          raise "fatal: fork returned nil" if pid.nil?
+          add_demon name, pid
         end
-        raise "fatal: fork returned nil" if pid.nil?
-        @demons << [name, pid]
-        DeepTest.logger.debug "Started #{name} (#{pid})"
+
+        launch_reaper_thread name, pid
+
       rescue => e
         puts "exception starting #{name}: #{e}"
         puts "\t" + e.backtrace.join("\n\t")
       end
     end
 
+    def demon_count
+      @demons_semaphore.synchronize do
+        @demons.size
+      end
+    end
+
     def stop_all
-      @demons.reverse.each do |demon|
+      DeepTest.logger.debug("stopping all demons")
+      receivers = @demons_semaphore.synchronize do
+        @demons.reverse
+      end
+
+      receivers.reverse.each do |demon|
         name, pid = demon
         if running?(pid)
-          Process.kill("HUP", pid)
+          DeepTest.logger.debug("Sending SIGTERM to #{name}, #{pid}")
+          Process.kill("TERM", pid)
         end
       end
-      @demons.each do |demon|
-        name, pid = demon
-        begin
-          Process.wait(pid)
-        rescue Errno::ECHILD => e
-          puts e
+      DeepTest.logger.debug("Warlock: Stopped all receivers")
+
+      DeepTest.logger.debug("waiting for reapers")
+      @reapers.each {|r| r.join}
+
+      DeepTest.logger.debug("Warlock: done reaping processes")
+    end
+
+    def exit_when_none_running
+      Thread.new do
+        loop do
+          Thread.pass
+          exit(0) unless any_running?
+          sleep(0.01)
         end
-        DeepTest.logger.debug "Stopped #{name} (#{pid})"
+      end
+    end
+
+    def any_running?
+      @demons_semaphore.synchronize do
+        @demons.any? {|name, pid| running?(pid)}
       end
     end
 
@@ -55,5 +106,29 @@ module DeepTest
       #  return false
       end
     end    
+
+    protected
+
+    def add_demon(name, pid)
+      DeepTest.logger.debug "Started: #{name} (#{pid})"
+      @demons << [name, pid]
+    end
+
+    def remove_demon(name, pid)
+      @demons.delete [name, pid]
+      DeepTest.logger.debug "Stopped: #{name} (#{pid})"
+    end
+
+
+    def launch_reaper_thread(name, pid)
+      @reapers << Thread.new do
+        Process.detach(pid).join
+        DeepTest.logger.debug("#{name} (#{pid}) reaped")
+        @demons_semaphore.synchronize do
+          DeepTest.logger.debug("Warlock Reaper: removing #{name} (#{pid}) from demon list")
+          remove_demon name, pid
+        end
+      end
+    end
   end
 end

data/lib/deep_test/worker.rb

@@ -10,11 +10,47 @@ module DeepTest
 
     def run
       @listener.starting(self)
-      while work_unit = @blackboard.take_work
+      while work_unit = next_work_unit
         @listener.starting_work(self, work_unit)
-        result = work_unit.run
+
+        result = begin
+                   work_unit.run
+                 rescue Exception => error
+                   Error.new(work_unit, error)
+                 end
+
         @listener.finished_work(self, work_unit, result)
         @blackboard.write_result result
+        if ENV['DEEP_TEST_SHOW_WORKER_DOTS'] == 'yes'
+          $stdout.print '.'
+          $stdout.flush
+        end
+      end
+    rescue Server::NoWorkUnitsRemainingError
+      DeepTest.logger.debug("Worker #{number}: no more work to do")
+    end
+
+    def next_work_unit
+      @blackboard.take_work
+    rescue Server::NoWorkUnitsAvailableError
+      sleep 0.02
+      retry
+    end
+
+    class Error
+      attr_accessor :work_unit, :error
+
+      def initialize(work_unit, error)
+        @work_unit, @error = work_unit, error
+      end
+
+      def ==(other)
+        work_unit == other.work_unit &&
+            error == other.error
+      end
+
+      def to_s
+        "#{@work_unit}: #{@error}\n" + (@error.backtrace || []).join("\n")
       end
     end
   end

data/script/internal/run_test_suite.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + "/../../lib/deep_test"
+options = DeepTest::Options.from_command_line(ARGV[0])
+DeepTest.init(options)
+runner = DeepTest::Test::Runner.new(options)
+workers = options.new_workers
+workers.load_files Dir.glob(options.pattern)
+DeepTest::ProcessOrchestrator.run(options, workers, runner)

data/script/public/master_test_server.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + "/../../lib/deep_test"
+require 'drb/drb'
+require 'optparse'
+
+uri = "drubyall://:4021"
+slave_uris = OptionParser.new do |opts|
+  opts.banner = "Usage: deep_test master_test_server [options] <test_server_uris>"
+
+  opts.on("--uri URI", "DRb URI to bind server to") do |v|
+    uri = v
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse(ARGV)
+
+begin
+  DeepTest::Distributed::MasterTestServer.start(uri, slave_uris)
+rescue Interrupt
+  DeepTest.logger.info "Exiting due to Interrupt"
+end
+

data/script/public/test_server.rb

@@ -0,0 +1,18 @@
+require File.dirname(__FILE__) + "/../../lib/deep_test"
+require 'drb/drb'
+require 'optparse'
+
+config = DeepTest::Distributed::TestServer.parse_args(ARGV)
+
+#
+# Clear args so they won't be processed in any forked processes.
+#    - When specs are loaded by the RemoteWorkerServer, RSpec
+#      attempts to parse ARGV
+#
+ARGV.clear
+
+begin
+  DeepTest::Distributed::TestServer.start(config)
+rescue Interrupt
+  DeepTest.logger.info "Exiting due to Interrupt"
+end

data/script/public/test_throughput.rb

@@ -0,0 +1,29 @@
+require File.dirname(__FILE__) + '/../../lib/deep_test'
+
+unless ARGV.length == 2
+  puts "Usage: deep_test test_throughput <server uri> <test_count>"
+  exit(1)
+end
+
+uri        = ARGV[0]
+test_count = ARGV[1].to_i
+
+options = DeepTest::Options.new(:distributed_server => uri,
+                                :sync_options => {:source => ""})
+server  = DeepTest::Distributed::TestServer.connect options
+workers = DeepTest::Distributed::ThroughputWorkerClient.new(options, server)
+runner  = DeepTest::Distributed::ThroughputRunner.new(options, test_count) do |result|
+  $stdout.print "."
+  $stdout.flush
+end
+
+start_time = Time.now
+DeepTest::ProcessOrchestrator.new(options, workers, runner).run(false)
+end_time = Time.now
+
+puts
+puts runner.statistics.summary
+
+run_time = end_time.to_f - start_time.to_f
+puts "Total Run Time: #{run_time} seconds"
+puts "Run Time Not Spent On Tests: #{run_time - runner.statistics.timespan_in_seconds}"

data/test/deep_test/database/mysql_setup_listener_test.rb

@@ -0,0 +1,14 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+unit_tests do
+  test "dump_schema includes procedures" do
+    listener = DeepTest::Database::MysqlSetupListener.new
+    listener.expects(:system).with do |command|
+      command =~ / -R /
+    end
+    listener.expects(:master_database_config).returns({})
+    listener.expects(:dump_file_name).returns("")
+    $?.expects(:success?).returns(true)
+    listener.dump_schema
+  end
+end

data/test/deep_test/distributed/dispatch_controller_test.rb

@@ -0,0 +1,209 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+unit_tests do
+  test "dispatch invokes each receiver once" do
+    receiver_1, receiver_2 = mock, mock
+
+    controller = DeepTest::Distributed::DispatchController.new(
+      DeepTest::Options.new({:ui => "DeepTest::UI::Null"}), 
+      [receiver_1, receiver_2]
+    )
+
+    receiver_1.expects(:a_method).with(:args)
+    receiver_2.expects(:a_method).with(:args)
+
+    controller.dispatch(:a_method, :args)
+  end
+
+  test "dispatch returns array of all results" do
+    receiver_1, receiver_2 = mock, mock
+
+    controller = DeepTest::Distributed::DispatchController.new(
+      DeepTest::Options.new({:ui => "DeepTest::UI::Null"}), 
+      [receiver_1, receiver_2]
+    )
+
+    receiver_1.expects(:a_method).returns(:result_1)
+    receiver_2.expects(:a_method).returns(:result_2)
+
+    results = controller.dispatch(:a_method)
+    assert_equal 2, results.size
+    assert_equal [:result_1, :result_2].to_set, results.to_set
+  end
+
+  test "dispatch calls all receivers in parallel" do
+    options = DeepTest::Options.new({:ui => "DeepTest::UI::Null"})
+    waiter = Waiter.new(tracker = Tracker.new)
+
+    Timeout.timeout(1) do
+      DeepTest::Distributed::DispatchController.new(options,[tracker, waiter]).
+        dispatch(:tracked_method)
+    end
+
+    waiter = Waiter.new(tracker = Tracker.new)
+
+    Timeout.timeout(1) do
+      DeepTest::Distributed::DispatchController.new(options,[waiter, tracker]).
+        dispatch(:tracked_method)
+    end
+  end
+
+  test "dispatch omits results that are taking too long" do
+    receiver = Object.new
+    def receiver.__drburi; ""; end
+    def receiver.sleep_100_millis
+      sleep 0.1
+    end
+
+    controller = DeepTest::Distributed::DispatchController.new(
+      DeepTest::Options.new({:ui => "DeepTest::UI::Null", :timeout_in_seconds => 0.05}),
+      [receiver]
+    )
+
+    DeepTest.logger.expects(:error)
+
+    assert_equal [], controller.dispatch(:sleep_100_millis)
+  end
+
+  test "after timeout, no further calls are sent to that receiver" do
+    receiver_1, receiver_2 = mock(:__drburi => ""), mock
+    receiver_1.expects(:method_call_1).raises(Timeout::Error.new("message"))
+    receiver_1.expects(:method_call_2).never
+
+    receiver_2.expects(:method_call_1)
+    receiver_2.expects(:method_call_2)
+
+    controller = DeepTest::Distributed::DispatchController.new(
+      DeepTest::Options.new({:ui => "DeepTest::UI::Null", :timeout_in_seconds => 0.05}),
+      [receiver_1, receiver_2]
+    )
+
+    DeepTest.logger.expects(:error)
+    
+    controller.dispatch(:method_call_1)
+    controller.dispatch(:method_call_2)
+  end
+
+  test "receiver is dropped when connection is refused" do
+    receiver_1, receiver_2 = mock(:__drburi => ""), mock
+    receiver_1.expects(:method_call_1).raises(DRb::DRbConnError)
+    receiver_1.expects(:method_call_2).never
+
+    receiver_2.expects(:method_call_1).returns(:value)
+    receiver_2.expects(:method_call_2)
+
+    controller = DeepTest::Distributed::DispatchController.new(
+      DeepTest::Options.new({:ui => "DeepTest::UI::Null", :timeout_in_seconds => 0.05}),
+      [receiver_1, receiver_2]
+    )
+
+    DeepTest.logger.expects(:error)
+    
+    assert_equal [:value], controller.dispatch(:method_call_1)
+
+    controller.dispatch(:method_call_2)
+  end
+  
+  test "receiver is dropped when any exception occurs" do
+    receiver = mock(:__drburi => "")
+    receiver.expects(:method_call).raises(Exception)
+
+    controller = DeepTest::Distributed::DispatchController.new(
+      DeepTest::Options.new({:ui => "DeepTest::UI::Null", :timeout_in_seconds => 0.05}),
+      [receiver]
+    )
+
+    DeepTest.logger.expects(:error)
+    
+    controller.dispatch(:method_call)
+    assert_raises(DeepTest::Distributed::NoDispatchReceiversError) {controller.dispatch(:another_call)}
+  end
+
+  test "no error is printed if dispatching without error" do
+    receiver_1 =  mock
+    receiver_1.expects(:method_call_1).raises(DRb::DRbConnError)
+
+    controller = DeepTest::Distributed::DispatchController.new(
+      DeepTest::Options.new({:ui => "DeepTest::UI::Null", :timeout_in_seconds => 0.05}),
+      [receiver_1]
+    )
+
+    DeepTest.logger.expects(:error).never
+
+    controller.dispatch_with_options(:method_call_1, :ignore_connection_error => true)
+  end
+
+  test "dispatch calls notifies ui of start and stop of dispatch" do
+    options = DeepTest::Options.new({:ui => "DeepTest::UI::Null"})
+    controller = DeepTest::Distributed::DispatchController.new(
+      options, [stub_everything]
+    )
+
+    options.ui_instance.expects(:dispatch_starting).with(:method_name)
+    options.ui_instance.expects(:dispatch_finished).with(:method_name)
+    
+    controller.dispatch(:method_name)
+  end
+
+  test "dispatch calls notifies ui dispatch end in case of an error" do
+    options = DeepTest::Options.new({:ui => "DeepTest::UI::Null"})
+    controller = DeepTest::Distributed::DispatchController.new(
+      options, [receiver = mock(:__drburi => '')]
+    )
+    receiver.expects(:method_name).raises("An Error")
+    DeepTest.logger.expects(:error)
+    
+    options.ui_instance.expects(:dispatch_starting).with(:method_name)
+    options.ui_instance.expects(:dispatch_finished).with(:method_name)
+    
+    begin
+      controller.dispatch(:method_name)
+    rescue RuntimeError => e
+      raise unless e.message == "An Error"
+    end
+  end
+
+  test "error is raised if dispatch to no receivers in attempted" do
+    options = DeepTest::Options.new({:ui => "DeepTest::UI::Null"})
+    controller = DeepTest::Distributed::DispatchController.new(
+      options, []
+    )
+
+    assert_raises(DeepTest::Distributed::NoDispatchReceiversError) do
+      controller.dispatch(:any_method)
+    end
+  end
+
+  class Tracker
+    def initialize
+      @mutex = Mutex.new
+      @condvar = ConditionVariable.new
+    end
+
+    def tracked_method 
+      @mutex.synchronize do
+        @tracked_method_called = true
+        @condvar.broadcast
+      end
+    end
+
+    def wait_for_tracked_method
+      @mutex.synchronize do
+        loop do
+          return if @tracked_method_called
+          @condvar.wait(@mutex)
+        end
+      end
+    end
+  end
+
+  class Waiter
+    def initialize(tracker)
+      @tracker = tracker
+    end
+
+    def tracked_method
+      @tracker.wait_for_tracked_method
+    end
+  end
+end