deep_test_pre 2.0

data/lib/deep_test/deadlock_detector.rb

@@ -0,0 +1,7 @@
+module DeepTest
+  class DeadlockDetector
+    def self.due_to_deadlock?(error)
+      error && !error.message.to_s.match(/Deadlock found when trying to get lock/).nil?
+    end
+  end
+end

data/lib/deep_test/demon.rb

@@ -0,0 +1,25 @@
+module DeepTest
+  module Demon
+    def forked(name, options, demon_args)
+      options.connect_to_central_command do |wire|
+        ProxyIO.replace_stdout_stderr!(wire) do
+          begin
+            catch(:exit_demon) do
+              Signal.trap("TERM") { throw :exit_demon }
+              execute *demon_args
+            end
+          rescue SystemExit => e
+            raise
+          rescue Exception => e
+            FailureMessage.show self.class.name, "Process #{Process.pid} exiting with excetion: #{e.class}: #{e.message}"
+            raise
+          end
+        end
+      end
+    end
+
+    def execute(*args)
+      raise "#{self.class} must implement the execute method to be a Demon"
+    end
+  end
+end

data/lib/deep_test/distributed/beachhead.rb

@@ -0,0 +1,104 @@
+module DeepTest
+  module Distributed
+    class Beachhead < LocalDeployment
+      include Demon
+
+      MERCY_KILLING_GRACE_PERIOD = 10 * 60 unless defined?(MERCY_KILLING_GRACE_PERIOD)
+
+      def initialize(base_path, options)
+        super options
+        @base_path = base_path
+      end
+
+      def launch_mercy_killer(grace_period)
+        Thread.new do
+          sleep grace_period
+          exit(0) unless agents_deployed?
+        end
+      end
+
+      def load_files(files)
+        spec_support_path = File.expand_path(File.dirname(__FILE__) + "/../spec") 
+        Dir.chdir @base_path
+        resolver = FilenameResolver.new(@base_path)
+        files.each do |file|
+          load resolver.resolve(file)
+        end
+
+        # Load rspec support if rspec is available now that we've loaded the host project files
+        #
+        DeepTest::RSpecDetector.if_rspec_available do
+          require spec_support_path
+        end
+      end
+
+      def deploy_agents
+        @agents_deployed = true
+        super
+        warlock.exit_when_none_running
+      end
+
+      def agents_deployed?
+        @agents_deployed
+      end
+
+      def forked(*args)
+        $stdout.reopen("/dev/null")
+        $stderr.reopen("/dev/null")
+        super
+      end
+
+      def execute(innie, outie, grace_period)
+        innie.close
+
+        switchboard = Telegraph::Switchboard.new
+        operator = Telegraph::Operator.listen "0.0.0.0", 0, switchboard
+
+        DeepTest.logger.debug { "Beachhead started on port #{operator.port}" }
+
+        outie.write operator.port
+        outie.close
+
+        launch_mercy_killer grace_period
+
+        loop do
+          begin
+            switchboard.process_messages :timeout => 1 do |message, wire|
+              case message.body
+              when LoadFiles
+                load_files message.body.files
+              when DeployAgents
+                deploy_agents
+                wire.send_message Done
+                operator.shutdown
+                break
+              end
+            end
+          end
+        end
+      end
+
+      def daemonize(grace_period = MERCY_KILLING_GRACE_PERIOD)
+        innie, outie = IO.pipe
+
+        warlock.start "Beachhead", self, innie, outie, grace_period
+
+        outie.close
+        port = innie.gets
+        innie.close
+        port.to_i
+      end
+
+      unless defined? DeployAgents
+        DeployAgents = "DeployAgents"
+        Done = "Done"
+        class LoadFiles
+          attr_reader :files
+          def initialize(files)
+            @files = files
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/dispatch_controller.rb

@@ -0,0 +1,60 @@
+module DeepTest
+  module Distributed
+    class DispatchController
+      def initialize(options, receivers)
+        @options = options
+        @receivers = receivers
+      end
+
+      def dispatch(method_name, *args)
+        dispatch_with_options(method_name, {}, *args)
+      end
+
+      def dispatch_with_options(method_name, options, *args)
+        raise NoDispatchReceiversError if @receivers.empty?
+
+        @options.ui_instance.dispatch_starting(method_name)
+
+        threads = @receivers.map do |r|
+          Thread.new do
+            begin
+              Thread.current[:receiver] = r
+              r.send method_name, *args
+            rescue Exception => e
+              Thread.current[:original_exception] = e
+              raise
+            end
+          end
+        end
+
+        results = []
+        threads.each do |t|
+          begin
+            results << t.value
+          rescue Timeout::Error
+            @receivers.delete t[:receiver]
+            DeepTest.logger.error { "Timeout dispatching #{method_name} to #{description t[:receiver]}" }
+          rescue Exception => this_exception
+            @receivers.delete t[:receiver]
+            DeepTest.logger.error { "Exception while dispatching #{method_name} to #{description t[:receiver]}:" }
+
+            e = t[:original_exception] || this_exception 
+            DeepTest.logger.error { "#{e.class}: #{e.message}" }
+            e.backtrace.each {|l| DeepTest.logger.error { l } }
+          end
+        end
+
+        results
+      ensure
+        @options.ui_instance.dispatch_finished(method_name)
+      end
+
+      def description(receiver)
+        receiver.inspect
+      end
+
+    end
+
+    class NoDispatchReceiversError < StandardError; end
+  end
+end

data/lib/deep_test/distributed/establish_beachhead.rb

@@ -0,0 +1,19 @@
+require File.dirname(__FILE__) + "/../../deep_test"
+options = DeepTest::Options.from_command_line(ENV['OPTIONS'])
+ENV['DEEP_TEST_LOG_LEVEL'] = options.environment_log_level
+options.ssh_client_connection_info = DeepTest::Distributed::SshClientConnectionInfo.new
+
+DeepTest.logger.debug { "mirror establish_beachhead for #{options.origin_hostname}" }
+
+STDIN.close
+
+beachhead_port = DeepTest::Distributed::Beachhead.new(
+  File.join(options.mirror_path, File.basename(options.sync_options[:source])), 
+  options
+).daemonize
+
+puts "Beachhead port: #{beachhead_port}"
+$stdout.flush
+
+exit!(0)
+

data/lib/deep_test/distributed/filename_resolver.rb

@@ -0,0 +1,40 @@
+module DeepTest
+  module Distributed
+    class FilenameResolver
+      def initialize(base_path)
+        @base_path = base_path
+      end
+
+      def resolve(filename)
+        return resolve("/" + filename) unless filename[0] == ?/
+
+        return filename.sub(@cached_replaced_path, @base_path) if @cached_replaced_path
+
+        each_potential_filename(filename) do |potential_filename|
+          if File.exist?(potential_filename)
+            cache_resolution(filename, potential_filename)
+            return potential_filename 
+          end
+        end
+
+        raise "Filename resolution failed.  Cannot resolve #{filename} within #{@base_path}"
+      end
+
+      def cache_resolution(original_filename, resolved_filename)
+        @cached_replaced_path = original_filename.sub(
+           resolved_filename.sub(@base_path, ""), ""
+        )
+      end
+
+      def each_potential_filename(filename)
+        basename = File.basename(filename)
+        dirs = File.dirname(filename).split('/')
+
+        begin
+          dirs.shift
+          yield [@base_path, dirs, basename].flatten.join('/')
+        end until dirs.empty?
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/landing_fleet.rb

@@ -0,0 +1,30 @@
+module DeepTest
+  module Distributed
+    class LandingFleet
+      def initialize(options, slaves)
+        DeepTest.logger.debug { "LandingFleet#initialize #{slaves.length} slaves" }
+        @slave_controller = DispatchController.new(options, slaves)
+      end
+
+      def establish_beachhead(options)
+        DeepTest.logger.debug { "dispatch establish_beachhead for #{options.origin_hostname}" }
+        @slave_controller.dispatch :establish_beachhead, options
+      end
+
+      def push_code(options)
+        DeepTest.logger.debug { "dispatch push_code for #{options.origin_hostname}" }
+        @slave_controller.dispatch :push_code, options
+      end
+
+      def load_files(files)
+        DeepTest.logger.debug { "dispatch load_files" }
+        @slave_controller.dispatch :load_files, files
+      end
+
+      def deploy_agents
+        DeepTest.logger.debug { "dispatch deploy_agents" }
+        @slave_controller.dispatch :deploy_agents
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/landing_ship.rb

@@ -0,0 +1,60 @@
+module DeepTest
+  module Distributed
+    class LandingShip
+      def initialize(config)
+        @config = config
+      end
+
+      def push_code(options)
+        RSync.push(@config[:address], options.sync_options, options.mirror_path)
+      end
+
+      def establish_beachhead(options)
+        command  = "#{ssh_command(options)} '#{spawn_command(options)}' 2>&1"
+        DeepTest.logger.debug { "Establishing Beachhead: #{command}" }
+        
+        output = `#{command}`
+        output.each do |line|
+          if DeepTest.logger.level == Logger::DEBUG
+            puts output
+          end
+          if line =~ /Beachhead port: (.+)/
+            @wire = Telegraph::Wire.connect(@config[:address], $1.to_i)
+          end
+        end
+        raise "LandingShip unable to establish Beachhead.  Output from #{@config[:address]} was:\n#{output}" unless @wire
+      end
+
+      def load_files(files)
+        @wire.send_message Beachhead::LoadFiles.new(files)
+      end
+
+      def deploy_agents
+        @wire.send_message Beachhead::DeployAgents
+        begin
+          message = @wire.next_message :timeout => 1
+          raise "Unexpected message from Beachhead: #{message.inspect}" unless message.body == Beachhead::Done
+        rescue Telegraph::NoMessageAvailable
+          retry
+        end
+      end
+
+      def ssh_command(options)
+        username_option = if options.sync_options[:username]
+                            " -l #{options.sync_options[:username]}"
+                          else
+                            ""
+                          end
+
+        "ssh -4 #{@config[:address]}#{username_option}"
+      end
+
+      def spawn_command(options)
+        "#{ShellEnvironment.like_login} && " + 
+        "cd #{options.mirror_path} && " + 
+        "OPTIONS=#{options.to_command_line} " + 
+        "ruby lib/deep_test/distributed/establish_beachhead.rb" 
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/remote_deployment.rb

@@ -0,0 +1,56 @@
+module DeepTest
+  module Distributed
+    class RemoteDeployment
+      def initialize(options, landing_fleet, failover_deployment)
+        @failover_deployment = failover_deployment
+        @options = options
+        @landing_fleet = landing_fleet
+      end
+
+      def load_files(filelist)
+        # load one file before calling listeners to make sure environment has
+        # been initialized as expected
+        #
+        load filelist.first
+        @options.new_listener_list.before_sync
+
+        t = Thread.new do
+          @landing_fleet.push_code(@options)
+          @landing_fleet.establish_beachhead(@options)
+          @landing_fleet.load_files filelist
+        end
+
+        filelist[1..-1].each {|f| load f}
+
+        begin
+          t.join
+        rescue => e
+          # The failover here doesn't invoke load_files on the failover_deployment
+          # because it will be a LocalDeployment, which forks from the current 
+          # process.  The fact that we depend in this here is damp...
+          #
+          fail_over("load_files", e)
+        end
+      end
+
+      def deploy_agents
+        DeepTest.logger.debug { "RemoteDeployment deploying agents with #{@landing_fleet}" }
+        @landing_fleet.deploy_agents
+      rescue => e
+        raise if failed_over?
+        fail_over("deploy_agents", e)
+        retry
+      end
+
+      def fail_over(method, exception)
+        DeepTest.logger.debug { "RemoteDeployment failing over on #{method}." }
+        @options.ui_instance.distributed_failover_to_local(method, exception)
+        @landing_fleet = @failover_deployment
+      end
+
+      def failed_over?
+        @landing_fleet == @failover_deployment
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/rsync.rb

@@ -0,0 +1,50 @@
+module DeepTest
+  module Distributed
+    class RSync
+      def self.push(address, sync_options, destination)
+        sync(:push, address, sync_options, destination)
+      end
+
+      def self.sync(operation, address, sync_options, destination)
+        command = Args.new(address, sync_options).send("#{operation}_command", 
+                                                  destination)
+
+        DeepTest.logger.debug { "rsycing: #{command}" }
+        successful = system command
+        raise "RSync Failed!!" unless successful
+      end
+
+      class Args
+        def initialize(address, sync_options)
+          @address = address
+          @sync_options = sync_options
+        end
+
+        def pull_command(destination)
+          command remote_location(@sync_options[:source]), destination
+        end
+
+        def push_command(destination)
+          command @sync_options[:source], remote_location(destination)
+        end
+
+        def command(source, destination)
+          # The '/' after source tells rsync to copy the contents
+          # of source to destination, rather than the source directory
+          # itself
+          "rsync -az --delete #{@sync_options[:rsync_options]} #{DeepTest::LIB_ROOT} #{source} #{destination}".strip.squeeze(" ")
+        end
+
+        def remote_location(path)
+          source = ""
+          unless @sync_options[:local]
+            source << @sync_options[:username] << '@' if @sync_options[:username]
+            source << @address
+            source << (@sync_options[:daemon] ? '::' : ':')
+          end
+          source << path
+        end
+      end
+    end
+  end
+end