jperkins-deep_test 1.2.2

data/lib/deep_test/deadlock_detector.rb

@@ -0,0 +1,7 @@
+module DeepTest
+  class DeadlockDetector
+    def self.due_to_deadlock?(error)
+      error && !error.message.to_s.match(/Deadlock found when trying to get lock/).nil?
+    end
+  end
+end

data/lib/deep_test/distributed/dispatch_controller.rb

@@ -0,0 +1,53 @@
+module DeepTest
+  module Distributed
+    class DispatchController
+      def initialize(options, receivers)
+        @options = options
+        @receivers = receivers
+      end
+
+      def dispatch(method_name, *args)
+        dispatch_with_options(method_name, {}, *args)
+      end
+
+      def dispatch_with_options(method_name, options, *args)
+        raise NoDispatchReceiversError if @receivers.empty?
+
+        @options.ui_instance.dispatch_starting(method_name)
+
+        threads = @receivers.map do |r|
+          Thread.new do
+            Thread.current[:receiver] = r
+            Timeout.timeout(@options.timeout_in_seconds) do
+              r.send method_name, *args
+            end
+          end
+        end
+
+        results = []
+        threads.each do |t|
+          begin
+            results << t.value
+          rescue Timeout::Error
+            @receivers.delete t[:receiver]
+            DeepTest.logger.error "Timeout dispatching #{method_name} to #{t[:receiver].__drburi}"
+          rescue DRb::DRbConnError
+            @receivers.delete t[:receiver]
+            unless options[:ignore_connection_error]
+              DeepTest.logger.error "Connection Refused dispatching #{method_name} to #{t[:receiver].__drburi}"
+            end
+          rescue Exception => e
+            @receivers.delete t[:receiver]
+            DeepTest.logger.error "Exception while dispatching #{method_name} to #{t[:receiver].__drburi} #{e.message}"
+          end
+        end
+
+        results
+      ensure
+        @options.ui_instance.dispatch_finished(method_name)
+      end
+    end
+
+    class NoDispatchReceiversError < StandardError; end
+  end
+end

data/lib/deep_test/distributed/drb_client_connection_info.rb

@@ -0,0 +1,15 @@
+module DeepTest
+  module Distributed
+    class DRbClientConnectionInfo
+      attr_reader :address
+
+      def initialize
+        info = Thread.current['DRb']
+        raise "No DRb client found" unless info && info['client']
+        peeraddr = info['client'].peeraddr
+        DeepTest.logger.debug("DRbClientConnection info: #{peeraddr.inspect}")
+        @address = peeraddr[3]
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/filename_resolver.rb

@@ -0,0 +1,40 @@
+module DeepTest
+  module Distributed
+    class FilenameResolver
+      def initialize(base_path)
+        @base_path = base_path
+      end
+
+      def resolve(filename)
+        return resolve("/" + filename) unless filename[0] == ?/
+
+        return filename.sub(@cached_replaced_path, @base_path) if @cached_replaced_path
+
+        each_potential_filename(filename) do |potential_filename|
+          if File.exist?(potential_filename)
+            cache_resolution(filename, potential_filename)
+            return potential_filename 
+          end
+        end
+
+        raise "Filename resolution failed.  Cannot resolve #{filename} within #{@base_path}"
+      end
+
+      def cache_resolution(original_filename, resolved_filename)
+        @cached_replaced_path = original_filename.sub(
+           resolved_filename.sub(@base_path, ""), ""
+        )
+      end
+
+      def each_potential_filename(filename)
+        basename = File.basename(filename)
+        dirs = File.dirname(filename).split('/')
+
+        begin
+          dirs.shift
+          yield [@base_path, dirs, basename].flatten.join('/')
+        end until dirs.empty?
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/master_test_server.rb

@@ -0,0 +1,52 @@
+module DeepTest
+  module Distributed
+    class MasterTestServer
+      include ERB::Util
+
+      STATUS_PORT = 4020 unless defined?(STATUS_PORT)
+
+      attr_reader :servers
+
+      def initialize(servers)
+        @servers = servers
+      end
+
+      def show_status(req, res)
+        template = File.read(File.dirname(__FILE__) + "/show_status.rhtml")
+        res.body = ERB.new(template).result(binding)
+      end
+
+      def test_server_statuses
+        @servers.map do |s|
+          status = begin
+                     s.status
+                   rescue Exception => e
+                     e
+                   end
+
+          [s.__drburi, status]
+        end
+      end
+
+      def self.start(uri, server_uris)
+        master = start_drb(uri, server_uris)
+        start_http(master)
+        DeepTest.logger.info "MasterTestServer listening at #{DRb.uri}"
+        DRb.thread.join
+      end
+
+      def self.start_drb(uri, server_uris)
+        servers = server_uris.map {|server_uri| DRbObject.new_with_uri server_uri}
+        master = DeepTest::Distributed::MasterTestServer.new(servers)
+        DRb.start_service(uri, master)
+        master
+      end
+
+      def self.start_http(master)
+        s = WEBrick::HTTPServer.new :Port => STATUS_PORT
+        s.mount_proc("/", &master.method(:show_status))
+        Thread.new {s.start}
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/multi_test_server_proxy.rb

@@ -0,0 +1,44 @@
+module DeepTest
+  module Distributed
+    class MultiTestServerProxy
+      def initialize(options, slaves)
+        DeepTest.logger.debug "MultiTestServerProxy#initialize #{slaves.length} slaves"
+        @slave_controller = DispatchController.new(options, slaves)
+      end
+
+      def spawn_worker_server(options)
+        DeepTest.logger.debug "dispatch spawn_worker_server for #{options.origin_hostname}"
+        WorkerServerProxy.new options,
+                              @slave_controller.dispatch(:spawn_worker_server, 
+                                                         options)
+      end
+
+      def sync(options)
+        DeepTest.logger.debug "dispatch sync for #{options.origin_hostname}"
+        @slave_controller.dispatch(:sync, options)
+      end
+
+      class WorkerServerProxy
+        def initialize(options, slaves)
+          DeepTest.logger.debug "WorkerServerProxy#initialize #{slaves.inspect}"
+          @slave_controller = DispatchController.new(options, slaves)
+        end
+
+        def load_files(files)
+          DeepTest.logger.debug "dispatch load_files"
+          @slave_controller.dispatch(:load_files, files)
+        end
+
+        def start_all
+          DeepTest.logger.debug "dispatch start_all"
+          @slave_controller.dispatch(:start_all)
+        end
+
+        def stop_all
+          DeepTest.logger.debug "dispatch stop_all"
+          @slave_controller.dispatch_with_options(:stop_all, :ignore_connection_error => true)
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/null_work_unit.rb

@@ -0,0 +1,12 @@
+module DeepTest
+  module Distributed
+    #
+    # Work Unit used to measure throughput of workers.
+    #
+    class NullWorkUnit
+      def run
+        :null_work_unit_result
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/remote_worker_client.rb

@@ -0,0 +1,54 @@
+module DeepTest
+  module Distributed
+    class RemoteWorkerClient
+      def initialize(options, test_server, failover_workers)
+        @failover_workers = failover_workers
+        @options = options
+        @test_server = test_server
+      end
+
+      def load_files(filelist)
+        @options.new_listener_list.before_sync
+
+        t = Thread.new do
+          @test_server.sync(@options)
+          @worker_server = @test_server.spawn_worker_server(@options)
+          @worker_server.load_files filelist
+        end
+
+        filelist.each {|f| load f}
+
+        begin
+          t.join
+        rescue => e
+          # The failover here doesn't invoke load_files on the failover_workers
+          # because they will be LocalWorkers, which fork from the current 
+          # process.  The fact that we depend in this here is damp...
+          #
+          fail_over("load_files", e)
+        end
+      end
+
+      def start_all
+        @worker_server.start_all
+      rescue => e
+        raise if failed_over?
+        fail_over("start_all", e)
+        retry
+      end
+
+      def stop_all
+        @worker_server.stop_all
+      end
+
+      def fail_over(method, exception)
+        @options.ui_instance.distributed_failover_to_local(method, exception)
+        @worker_server = @failover_workers
+      end
+
+      def failed_over?
+        @worker_server == @failover_workers
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/remote_worker_server.rb

@@ -0,0 +1,82 @@
+module DeepTest
+  module Distributed
+    class RemoteWorkerServer
+      include DRb::DRbUndumped
+
+      MERCY_KILLING_GRACE_PERIOD = 10 * 60 unless defined?(MERCY_KILLING_GRACE_PERIOD)
+
+      def initialize(base_path, workers)
+        @base_path = base_path
+        @workers = workers
+      end
+
+      def launch_mercy_killer(grace_period)
+        Thread.new do
+          sleep grace_period
+          exit(0) unless workers_started?
+        end
+      end
+
+      def load_files(files)
+        Dir.chdir @base_path
+        resolver = FilenameResolver.new(@base_path)
+        files.each do |file|
+          load resolver.resolve(file)
+        end
+      end
+
+      def start_all
+        @workers_started = true
+        @workers.start_all
+      end
+
+      def stop_all
+        Thread.new do
+          @workers.stop_all
+        end
+      end
+
+      def workers_started?
+        @workers_started
+      end
+
+      def self.warlock
+        @warlock ||= DeepTest::Warlock.new
+      end
+
+      def self.running_server_count
+        @warlock.demon_count if @warlock
+      end
+
+      def self.stop_all
+        @warlock.stop_all if @warlock
+      end
+
+      def self.start(address, base_path, workers, grace_period = MERCY_KILLING_GRACE_PERIOD)
+        innie, outie = IO.pipe
+
+        warlock.start("RemoteWorkerServer") do
+          innie.close
+
+          server = new(base_path, workers)
+
+          DRb.start_service("drubyall://#{address}:0", server)
+          DeepTest.logger.info "RemoteWorkerServer started at #{DRb.uri}"
+
+          outie.write DRb.uri
+          outie.close
+
+          server.launch_mercy_killer(grace_period)
+
+          DRb.thread.join
+        end
+
+        outie.close
+        uri = innie.gets
+        innie.close
+        DRbObject.new_with_uri(uri)
+      end
+
+    end
+  end
+end

data/lib/deep_test/distributed/rsync.rb

@@ -0,0 +1,37 @@
+module DeepTest
+  module Distributed
+    class RSync
+      def self.sync(connection_info, options, destination)
+        command = Args.new(connection_info, options).command(destination)
+        DeepTest.logger.debug("rsycing: #{command}")
+        successful = system command
+        raise "RSync Failed!!" unless successful
+      end
+
+      class Args
+        def initialize(connection_info, options)
+          @connection_info = connection_info
+          @options = options
+          @sync_options = options.sync_options
+        end
+
+        def command(destination)
+          # The '/' after source tells rsync to copy the contents
+          # of source to destination, rather than the source directory
+          # itself
+          "rsync -az --delete #{@sync_options[:rsync_options]} #{source_location}/ #{destination}".strip.squeeze(" ")
+        end
+
+        def source_location
+          source = ""
+          unless @sync_options[:local]
+            source << @sync_options[:username] << '@' if @sync_options[:username]
+            source << @connection_info.address
+            source << (@sync_options[:daemon] ? '::' : ':')
+          end
+          source << @sync_options[:source]
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/show_status.rhtml

@@ -0,0 +1,41 @@
+<html>
+  <head>
+    <title>DeepTest Test Server Status</title>
+    <style>
+      .ok {
+        background-color: #AAFFAA;
+      }
+
+      .error {
+        background-color: #FFAAAA;
+      }
+
+    </style>
+  </head>
+  <body>
+    <h1>DeepTest Test Server Status</h1>
+    <table>
+      <tr>
+        <th>Mirror Server</th>
+        <th>Bound To</th>
+        <th>Workers / Client</th>
+        <th>Clients</th>
+      </tr>
+      <% test_server_statuses.each do |uri, status|
+           if Exception === status %>
+             <tr class="error">
+               <td><%=h uri %></td>
+               <td colspan="3"><%=h status.message %></td>
+             </tr>
+        <% else %>
+             <tr class="ok">
+               <td><%=h uri %></td>
+               <td><%=h status.binding_uri %></td>
+               <td><%=h status.number_of_workers %></td>
+               <td><%=h status.remote_worker_server_count %></td>
+             </tr>
+       <% end
+         end %>
+    </table>
+  </body>
+</html>