jstorimer-deep-test 1.4.0 → 2.0.0

data/lib/deep_test/distributed/dispatch_controller.rb

@@ -17,9 +17,12 @@ module DeepTest
 
         threads = @receivers.map do |r|
           Thread.new do
-            Thread.current[:receiver] = r
-            Timeout.timeout(@options.timeout_in_seconds) do
+            begin
+              Thread.current[:receiver] = r
               r.send method_name, *args
+            rescue Exception => e
+              Thread.current[:original_exception] = e
+              raise
             end
           end
         end
@@ -30,15 +33,14 @@ module DeepTest
             results << t.value
           rescue Timeout::Error
             @receivers.delete t[:receiver]
-            DeepTest.logger.error "Timeout dispatching #{method_name} to #{t[:receiver].__drburi}"
-          rescue DRb::DRbConnError
+            DeepTest.logger.error { "Timeout dispatching #{method_name} to #{description t[:receiver]}" }
+          rescue Exception => this_exception
             @receivers.delete t[:receiver]
-            unless options[:ignore_connection_error]
-              DeepTest.logger.error "Connection Refused dispatching #{method_name} to #{t[:receiver].__drburi}"
-            end
-          rescue Exception => e
-            @receivers.delete t[:receiver]
-            DeepTest.logger.error "Exception while dispatching #{method_name} to #{t[:receiver].__drburi} #{e.message}"
+            DeepTest.logger.error { "Exception while dispatching #{method_name} to #{description t[:receiver]}:" }
+
+            e = t[:original_exception] || this_exception 
+            DeepTest.logger.error { "#{e.class}: #{e.message}" }
+            e.backtrace.each {|l| DeepTest.logger.error { l } }
           end
         end
 
@@ -46,6 +48,11 @@ module DeepTest
       ensure
         @options.ui_instance.dispatch_finished(method_name)
       end
+
+      def description(receiver)
+        receiver.inspect
+      end
+
     end
 
     class NoDispatchReceiversError < StandardError; end

data/lib/deep_test/distributed/establish_beachhead.rb

@@ -0,0 +1,19 @@
+require File.dirname(__FILE__) + "/../../deep_test"
+options = DeepTest::Options.from_command_line(ENV['OPTIONS'])
+ENV['DEEP_TEST_LOG_LEVEL'] = options.environment_log_level
+options.ssh_client_connection_info = DeepTest::Distributed::SshClientConnectionInfo.new
+
+DeepTest.logger.debug { "mirror establish_beachhead for #{options.origin_hostname}" }
+
+STDIN.close
+
+beachhead_port = DeepTest::Distributed::Beachhead.new(
+  File.join(options.mirror_path, File.basename(options.sync_options[:source])), 
+  options
+).daemonize
+
+puts "Beachhead port: #{beachhead_port}"
+$stdout.flush
+
+exit!(0)
+

data/lib/deep_test/distributed/landing_fleet.rb

@@ -0,0 +1,30 @@
+module DeepTest
+  module Distributed
+    class LandingFleet
+      def initialize(options, slaves)
+        DeepTest.logger.debug { "LandingFleet#initialize #{slaves.length} slaves" }
+        @slave_controller = DispatchController.new(options, slaves)
+      end
+
+      def establish_beachhead(options)
+        DeepTest.logger.debug { "dispatch establish_beachhead for #{options.origin_hostname}" }
+        @slave_controller.dispatch :establish_beachhead, options
+      end
+
+      def push_code(options)
+        DeepTest.logger.debug { "dispatch push_code for #{options.origin_hostname}" }
+        @slave_controller.dispatch :push_code, options
+      end
+
+      def load_files(files)
+        DeepTest.logger.debug { "dispatch load_files" }
+        @slave_controller.dispatch :load_files, files
+      end
+
+      def deploy_agents
+        DeepTest.logger.debug { "dispatch deploy_agents" }
+        @slave_controller.dispatch :deploy_agents
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/landing_ship.rb

@@ -0,0 +1,60 @@
+module DeepTest
+  module Distributed
+    class LandingShip
+      def initialize(config)
+        @config = config
+      end
+
+      def push_code(options)
+        RSync.push(@config[:address], options.sync_options, options.mirror_path)
+      end
+
+      def establish_beachhead(options)
+        command  = "#{ssh_command(options)} '#{spawn_command(options)}' 2>&1"
+        DeepTest.logger.debug { "Establishing Beachhead: #{command}" }
+        
+        output = `#{command}`
+        output.each do |line|
+          if DeepTest.logger.level == Logger::DEBUG
+            puts output
+          end
+          if line =~ /Beachhead port: (.+)/
+            @wire = Telegraph::Wire.connect(@config[:address], $1.to_i)
+          end
+        end
+        raise "LandingShip unable to establish Beachhead.  Output from #{@config[:address]} was:\n#{output}" unless @wire
+      end
+
+      def load_files(files)
+        @wire.send_message Beachhead::LoadFiles.new(files)
+      end
+
+      def deploy_agents
+        @wire.send_message Beachhead::DeployAgents
+        begin
+          message = @wire.next_message :timeout => 1
+          raise "Unexpected message from Beachhead: #{message.inspect}" unless message.body == Beachhead::Done
+        rescue Telegraph::NoMessageAvailable
+          retry
+        end
+      end
+
+      def ssh_command(options)
+        username_option = if options.sync_options[:username]
+                            " -l #{options.sync_options[:username]}"
+                          else
+                            ""
+                          end
+
+        "ssh -4 #{@config[:address]}#{username_option}"
+      end
+
+      def spawn_command(options)
+        "#{ShellEnvironment.like_login} && " + 
+        "cd #{options.mirror_path} && " + 
+        "OPTIONS=#{options.to_command_line} " + 
+        "ruby lib/deep_test/distributed/establish_beachhead.rb" 
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/remote_deployment.rb

@@ -0,0 +1,56 @@
+module DeepTest
+  module Distributed
+    class RemoteDeployment
+      def initialize(options, landing_fleet, failover_deployment)
+        @failover_deployment = failover_deployment
+        @options = options
+        @landing_fleet = landing_fleet
+      end
+
+      def load_files(filelist)
+        # load one file before calling listeners to make sure environment has
+        # been initialized as expected
+        #
+        load filelist.first
+        @options.new_listener_list.before_sync
+
+        t = Thread.new do
+          @landing_fleet.push_code(@options)
+          @landing_fleet.establish_beachhead(@options)
+          @landing_fleet.load_files filelist
+        end
+
+        filelist[1..-1].each {|f| load f}
+
+        begin
+          t.join
+        rescue => e
+          # The failover here doesn't invoke load_files on the failover_deployment
+          # because it will be a LocalDeployment, which forks from the current 
+          # process.  The fact that we depend in this here is damp...
+          #
+          fail_over("load_files", e)
+        end
+      end
+
+      def deploy_agents
+        DeepTest.logger.debug { "RemoteDeployment deploying agents with #{@landing_fleet}" }
+        @landing_fleet.deploy_agents
+      rescue => e
+        raise if failed_over?
+        fail_over("deploy_agents", e)
+        retry
+      end
+
+      def fail_over(method, exception)
+        DeepTest.logger.debug { "RemoteDeployment failing over on #{method}." }
+        @options.ui_instance.distributed_failover_to_local(method, exception)
+        @landing_fleet = @failover_deployment
+      end
+
+      def failed_over?
+        @landing_fleet == @failover_deployment
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/rsync.rb

@@ -1,35 +1,48 @@
 module DeepTest
   module Distributed
     class RSync
-      def self.sync(connection_info, options, destination)
-        command = Args.new(connection_info, options).command(destination)
-        DeepTest.logger.debug("rsycing: #{command}")
+      def self.push(address, sync_options, destination)
+        sync(:push, address, sync_options, destination)
+      end
+
+      def self.sync(operation, address, sync_options, destination)
+        command = Args.new(address, sync_options).send("#{operation}_command", 
+                                                  destination)
+
+        DeepTest.logger.debug { "rsycing: #{command}" }
         successful = system command
         raise "RSync Failed!!" unless successful
       end
 
       class Args
-        def initialize(connection_info, options)
-          @connection_info = connection_info
-          @options = options
-          @sync_options = options.sync_options
+        def initialize(address, sync_options)
+          @address = address
+          @sync_options = sync_options
+        end
+
+        def pull_command(destination)
+          command remote_location(@sync_options[:source]), destination
+        end
+
+        def push_command(destination)
+          command @sync_options[:source], remote_location(destination)
         end
 
-        def command(destination)
+        def command(source, destination)
           # The '/' after source tells rsync to copy the contents
           # of source to destination, rather than the source directory
           # itself
-          "rsync -az --delete #{@sync_options[:rsync_options]} #{source_location}/ #{destination}".strip.squeeze(" ")
+          "rsync -az --delete #{@sync_options[:rsync_options]} #{DeepTest::LIB_ROOT} #{source} #{destination}".strip.squeeze(" ")
         end
 
-        def source_location
+        def remote_location(path)
           source = ""
           unless @sync_options[:local]
             source << @sync_options[:username] << '@' if @sync_options[:username]
-            source << @connection_info.address
+            source << @address
             source << (@sync_options[:daemon] ? '::' : ':')
           end
-          source << @sync_options[:source]
+          source << path
         end
       end
     end

data/lib/deep_test/distributed/shell_environment.rb

@@ -0,0 +1,50 @@
+module DeepTest
+  module Distributed
+    class ShellEnvironment
+      def self.like_login
+        login_env = new
+        login_env.include_first '/etc/profile'
+        login_env.include_first '~/.profile', '~/.bash_profile',  '~/.bashrc'
+        login_env
+      end
+
+      def initialize
+        @source_file_lists = []
+      end
+
+      def include_first(*filenames)
+        source_file_lists << SourceFileList.new(filenames)
+      end
+
+      def to_s
+        source_file_lists.join(" && ")
+      end
+
+      def ==(other)
+        source_file_lists == other.source_file_lists
+      end
+
+      attr_reader :source_file_lists
+      protected :source_file_lists
+
+      class SourceFileList
+        def initialize(filenames)
+          @filenames = filenames
+        end
+
+        def to_s
+          "if" + 
+          filenames.map {|f| " [[ -f #{f} ]]; then . #{f}; "}.join("elif") +
+          "fi"
+        end
+
+        def ==(other)
+          filenames == other.filenames
+        end
+
+        attr_reader :filenames
+        protected :filenames
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/ssh_client_connection_info.rb

@@ -0,0 +1,14 @@
+module DeepTest
+  module Distributed
+    class SshClientConnectionInfo
+      attr_reader :address
+
+      def initialize
+        raise "SSH_CLIENT environment variable not set" unless ENV['SSH_CLIENT']
+        ENV['SSH_CLIENT'] =~ /^(.+) \d+ \d+$/
+        raise "Unable to extract address from SSH_CLIENT (#{ENV['SSH_CLIENT']})" unless $1
+        @address = $1
+      end
+    end
+  end
+end

data/lib/deep_test/failure_message.rb

@@ -0,0 +1,19 @@
+module DeepTest
+  module FailureMessage
+    def self.show(title, message, width = 70)
+      lines = [" #{title} ".center(width, '*')]
+      message.each do |line|
+        lines << "* #{line.strip}".ljust(width - 1) + "*"
+      end
+      lines <<  "*" * width
+      string = lines.join("\n")
+      begin
+        puts string
+      rescue
+        IO.new(2) do |err|
+          err.puts string
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/lib_root.rb

@@ -0,0 +1,4 @@
+module DeepTest
+  LIB_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '/..')) unless defined?(DeepTest::LIB_ROOT)
+end
+

data/lib/deep_test/listener_list.rb

@@ -6,7 +6,7 @@ module DeepTest
       @listeners = listeners
     end
 
-    NullWorkerListener.instance_methods(false).each do |event|
+    NullListener.instance_methods(false).each do |event|
       eval <<-end_src
         def #{event}(*args)
           @listeners.each {|l| l.#{event}(*args)}

data/lib/deep_test/local_deployment.rb

@@ -0,0 +1,46 @@
+module DeepTest
+  class LocalDeployment
+    attr_reader :warlock
+
+    def initialize(options, agent_class = DeepTest::Agent)
+      @options = options
+      @agent_class = agent_class
+    end
+
+    def warlock
+      @warlock ||= Warlock.new @options
+    end
+
+    def load_files(files)
+      files.each {|f| load f}
+    end
+
+    def deploy_agents
+      DeepTest.logger.debug { "Deploying #{number_of_agents} #{@agent_class}s" }
+      wait_for_connect_threads = []
+      each_agent do |agent_num|
+        stream_from_child_process, stream_to_parent_process = IO.pipe
+        warlock.start "agent #{agent_num}", @agent_class.new(agent_num, @options, @options.new_listener_list), 
+                      stream_from_child_process, stream_to_parent_process
+        wait_for_connect_threads << Thread.new do
+          stream_to_parent_process.close
+          message = stream_from_child_process.read
+          stream_from_child_process.close
+          raise "Agent was not able to connect: #{message}" unless message == "Connected\n"
+        end
+      end        
+
+      wait_for_connect_threads.each { |t| t.join }
+    end
+
+    def number_of_agents
+      @options.number_of_agents
+    end
+
+    private
+
+    def each_agent
+      number_of_agents.to_i.times { |agent_num| yield agent_num }
+    end
+  end
+end

data/lib/deep_test/logger.rb

@@ -2,16 +2,31 @@ module DeepTest
   class Logger < ::Logger
     def initialize(*args)
       super
-      self.formatter = proc { |severity, time, progname, msg| "[DeepTest] #{msg}\n" }
+      hostname = Socket.gethostname
+      self.formatter = proc { |severity, time, progname, msg| "[DeepTest@#{hostname}] #{time.strftime "%F %T"} #{msg}\n" }
       self.level = configured_log_level
     end
 
+    def io_stream
+      @logdev.dev
+    end
+
     def configured_log_level
       if ENV['DEEP_TEST_LOG_LEVEL']
-        Logger.const_get(ENV['DEEP_TEST_LOG_LEVEL'])
+        Logger.const_get(ENV['DEEP_TEST_LOG_LEVEL'].upcase)
       else
         Logger::INFO
       end
     end
+
+    Severity.constants.each do |severity|
+      eval <<-end_src
+        def #{severity.downcase}
+          super
+        rescue Exception => e
+          super "\#{e.class}: \#{e} occurred logging on \#{caller[0]}", &nil
+        end
+      end_src
+    end
   end
 end