jstorimer-deep-test 1.4.0 → 2.0.0

data/lib/deep_test/central_command.rb

@@ -0,0 +1,165 @@
+require 'set'
+
+module DeepTest
+  class CentralCommand
+    attr_reader :operator
+    attr_reader :switchboard
+    attr_reader :data
+
+    def initialize(options)
+      @options = options
+      @work_queue = Queue.new
+      @results_mutex = Mutex.new
+      @results_condvar = ConditionVariable.new
+      @results = []
+      @data = Metrics::Data.new
+    end
+
+    def done_with_work
+      @done_with_work = true
+    end
+
+    def take_result
+      @results_mutex.synchronize do
+        loop do
+          if @results.any?
+            return @results.shift
+          else
+            @results_condvar.wait @results_mutex
+            raise NoAgentsRunningError unless @results.any? || @switchboard.any_live_wires?
+          end
+        end
+      end
+    end
+
+    def take_work
+      @work_queue.pop(true)
+    rescue ThreadError => e
+      if e.message == "queue empty"
+        raise NoWorkUnitsRemainingError if @done_with_work
+        raise NoWorkUnitsAvailableError
+      else
+        raise
+      end
+    end
+
+    def write_result(result)
+      @results_mutex.synchronize do
+        @results << result
+        @results_condvar.signal
+      end
+    end
+
+    def write_work(work_unit)
+      @work_queue.push work_unit
+    end
+
+    def stdout
+      $stdout
+    end
+
+    def stderr
+      $stderr
+    end
+
+    def self.start(options)
+      central_command = new(options)
+      central_command.start
+      central_command
+    end
+
+    def start
+      @switchboard = Telegraph::Switchboard.new
+      @operator = Telegraph::Operator.listen("0.0.0.0", 0, @switchboard)
+      @options.server_port = @operator.port
+      @process_messages_thread = Thread.new { process_messages }
+
+      DeepTest.logger.info { "Started DeepTest service on port #{@operator.port}" }
+    end
+
+    unless defined?(NeedWork)
+      NeedWork = "NeedWork" 
+      NoMoreWork = "NoMoreWork"
+      module Result; end
+      module Operation; end
+    end
+
+    def process_messages
+      loop do
+        begin
+          return if @stop_process_messages
+          wires_waiting_for_work.each { |w| send_work w }
+
+          @results_mutex.synchronize do
+            # make take_result wake up and check if any agents are running
+            @results_condvar.signal
+          end
+
+          message, wire = switchboard.next_message(:timeout => 0.5)
+
+          case message.body
+          when NeedWork
+            send_work wire
+          when Result 
+            write_result message.body
+            send_work wire
+          when Operation
+            message.body.execute
+          when Metrics::Measurement
+            data.add message.body
+          else 
+            raise UnexpectedMessageError, message.inspect
+          end
+
+        rescue Telegraph::NoMessageAvailable
+          retry
+        rescue Exception => e
+          raise unless @stop_process_messages
+        end
+      end
+    end
+
+    def wires_waiting_for_work
+      @wires_waiting_for_work ||= Set.new
+    end
+
+    def send_work(wire)
+      begin
+        wire.send_message take_work, :need_ack => true
+        wires_waiting_for_work.delete wire
+
+      rescue NoWorkUnitsAvailableError
+        put_abandonded_work_back_on_the_queue
+        wires_waiting_for_work.add wire
+
+      rescue NoWorkUnitsRemainingError
+        wire.send_message NoMoreWork
+        wires_waiting_for_work.delete wire
+
+      end
+    end
+
+    def put_abandonded_work_back_on_the_queue
+      closed_wires = switchboard.using_wires { |wires| wires.select {|w| w.closed? }}
+      closed_wires.each do |wire|
+        wire.unacked_messages.each do |m|
+          write_work m.body
+          switchboard.drop_wire wire
+        end
+      end
+    end
+
+    def stop
+      @stop_process_messages = true
+      operator.shutdown
+      @process_messages_thread.join
+      data.save @options.metrics_file if @options.gathering_metrics?
+    end
+
+    class NoWorkUnitsAvailableError < StandardError; end
+    class NoWorkUnitsRemainingError < StandardError; end
+    class NoAgentsRunningError < StandardError; end
+    class CheckIfAgentsAreStillRunning < StandardError; end
+    class UnexpectedMessageError < StandardError; end
+  end
+end

data/lib/deep_test/cpu_info.rb

@@ -0,0 +1,22 @@
+module DeepTest
+  class CpuInfo
+    attr_accessor :platform
+
+    def initialize(platform = RUBY_PLATFORM)
+      @platform = platform
+    end
+
+    def count
+      case platform
+      when /darwin/
+        output = `sysctl -n hw.ncpu`
+        output.strip.to_i
+      when /linux/
+        File.readlines("/proc/cpuinfo").inject(0) do |count, line|
+          next count + 1 if line =~ /processor\s*:\s*\d+/
+          count
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/database/mysql_setup_listener.rb

@@ -26,26 +26,25 @@ module DeepTest
       #
       def create_database
         admin_connection do |connection|
-          connection.create_database worker_database
+          connection.create_database agent_database
           grant_privileges connection
         end
       end
 
       #
-      # Grants 'all' privilege on worker database to username and password
-      # specified by worker database config.  If your application has
+      # Grants 'all' privilege on agent database to username and password
+      # specified by agent database config.  If your application has
       # special database privilege needs beyond 'all', you should override
       # this method and grant them.
       #
       def grant_privileges(connection)
-        puts "==== #{worker_database_config[:password].inspect}"
-        identified_by = if worker_database_config[:password]
-                          %{identified by %s} % connection.quote(worker_database_config[:password])
+        identified_by = if agent_database_config[:password]
+                          %{identified by %s} % connection.quote(agent_database_config[:password])
                         else
                           ""
                         end
-        sql = %{grant all on #{worker_database}.* to %s@'localhost' #{identified_by} ; } % 
-          connection.quote(worker_database_config[:username])
+        sql = %{grant all on #{agent_database}.* to %s@'localhost' #{identified_by} ; } % 
+          connection.quote(agent_database_config[:username])
 
         connection.execute sql
       end
@@ -55,7 +54,7 @@ module DeepTest
       #
       def drop_database
         admin_connection do |connection|
-          connection.drop_database worker_database
+          connection.drop_database agent_database
         end
       end
 
@@ -69,10 +68,10 @@ module DeepTest
       end
 
       #
-      # Loads dumpfile into worker database using mysql command
+      # Loads dumpfile into agent database using mysql command
       #
       def load_schema
-        config = command_line_config(worker_database_config)
+        config = command_line_config(agent_database_config)
         system "mysql #{config} < #{dump_file_name}"
         raise "Error Loading schema" unless $?.success?
       end
@@ -87,7 +86,7 @@ module DeepTest
       end
 
       def system(command) # :nodoc:
-        DeepTest.logger.info command
+        DeepTest.logger.info { command }
         super command
       end
 

data/lib/deep_test/database/setup_listener.rb

@@ -2,18 +2,18 @@ module DeepTest
   module Database
     #
     # Skeleton Listener to help with setting up a separate database
-    # for each worker.  Calls +dump_schema+, +load_schema+, +create_database+,
+    # for each agent.  Calls +dump_schema+, +load_schema+, +create_database+,
     # and +drop_database+ hooks provided by subclasses that implement database
     # setup strategies for particular database flavors.
     #
-    class SetupListener < NullWorkerListener
+    class SetupListener < NullListener
       DUMPED_SCHEMAS = [] unless defined?(DUMPED_SCHEMAS)
 
       def before_sync # :nodoc:
         dump_schema_once
       end
 
-      def before_starting_workers # :nodoc:
+      def before_starting_agents # :nodoc:
         dump_schema_once
       end
 
@@ -23,11 +23,11 @@ module DeepTest
         DUMPED_SCHEMAS << schema_name
       end
 
-      def starting(worker) # :nodoc:
-        @worker = worker
+      def starting(agent) # :nodoc:
+        @agent = agent
 
         at_exit do
-          DeepTest.logger.debug("dropping database #{worker_database}")
+          DeepTest.logger.debug { "dropping database #{agent_database}" }
           drop_database
         end
 
@@ -38,26 +38,26 @@ module DeepTest
       end
 
       #
-      # Called on each worker after creating database and before loading
+      # Called on each agent after creating database and before loading
       # schema to initialize connections
       # 
       def connect_to_database
-        ActiveRecord::Base.establish_connection(worker_database_config)
+        ActiveRecord::Base.establish_connection(agent_database_config)
       end
 
       #
-      # Called in each worker to create the database named by 
-      # +worker_database+.
+      # Called in each agent to create the database named by 
+      # +agent_database+.
       #
       def create_database
         raise "Subclass must implement"
       end
 
       #
-      # Called in each worker to drop the database created by
+      # Called in each agent to drop the database created by
       # +create_database+.  This method is called twice, once before
       # +create_database+ to ensure that no database exists and once
-      # at exit to clean as the worker process exits.  This method
+      # at exit to clean as the agent process exits.  This method
       # must not fail if the database does not exist when it is called.
       #
       def drop_database
@@ -65,12 +65,12 @@ module DeepTest
       end
 
       #
-      # Called before any workers are spawned to dump the schema that
+      # Called before any agents are spawned to dump the schema that
       # will be used for testing.  When running distributed, this method
       # is called on the local machine providing the tests to run.
       #
       # For distributed testing to work, the schema must be dumped in
-      # location accessible by all worker machines.  The easiest way to
+      # location accessible by all agent machines.  The easiest way to
       # accomplish this is to dump it to a location within the working copy.
       #
       def dump_schema
@@ -79,21 +79,21 @@ module DeepTest
 
 
       #
-      # Called once in each worker as it is starting to load the schema
+      # Called once in each agent as it is starting to load the schema
       # dumped from dump_schema.  Subclasses should load the schema definition
-      # into the +worker_database+
+      # into the +agent_database+
       #
       def load_schema
         raise "Subclass must implement"
       end
 
       #
-      # ActiveRecord configuration for the worker database.  By default,
+      # ActiveRecord configuration for the agent database.  By default,
       # the same as +master_database_config+, except that points to
-      # +worker_database+ instead of the database named in the master config.
+      # +agent_database+ instead of the database named in the master config.
       # 
-      def worker_database_config
-        master_database_config.merge(:database => worker_database)
+      def agent_database_config
+        master_database_config.merge(:database => agent_database)
       end
 
       #
@@ -106,10 +106,10 @@ module DeepTest
       end
 
       #
-      # Unique name for database on machine that worker is running on.
+      # Unique name for database on machine that agent is running on.
       #
-      def worker_database
-        "deep_test_worker_#{@worker.number}_pid_#{Process.pid}" 
+      def agent_database
+        "deep_test_agent_#{@agent.number}_pid_#{Process.pid}" 
       end
     end
   end

data/lib/deep_test/demon.rb

@@ -0,0 +1,25 @@
+module DeepTest
+  module Demon
+    def forked(name, options, demon_args)
+      options.connect_to_central_command do |wire|
+        ProxyIO.replace_stdout_stderr!(wire) do
+          begin
+            catch(:exit_demon) do
+              Signal.trap("TERM") { throw :exit_demon }
+              execute *demon_args
+            end
+          rescue SystemExit => e
+            raise
+          rescue Exception => e
+            FailureMessage.show self.class.name, "Process #{Process.pid} exiting with excetion: #{e.class}: #{e.message}"
+            raise
+          end
+        end
+      end
+    end
+
+    def execute(*args)
+      raise "#{self.class} must implement the execute method to be a Demon"
+    end
+  end
+end

data/lib/deep_test/distributed/beachhead.rb

@@ -0,0 +1,104 @@
+module DeepTest
+  module Distributed
+    class Beachhead < LocalDeployment
+      include Demon
+
+      MERCY_KILLING_GRACE_PERIOD = 10 * 60 unless defined?(MERCY_KILLING_GRACE_PERIOD)
+
+      def initialize(base_path, options)
+        super options
+        @base_path = base_path
+      end
+
+      def launch_mercy_killer(grace_period)
+        Thread.new do
+          sleep grace_period
+          exit(0) unless agents_deployed?
+        end
+      end
+
+      def load_files(files)
+        spec_support_path = File.expand_path(File.dirname(__FILE__) + "/../spec") 
+        Dir.chdir @base_path
+        resolver = FilenameResolver.new(@base_path)
+        files.each do |file|
+          load resolver.resolve(file)
+        end
+
+        # Load rspec support if rspec is available now that we've loaded the host project files
+        #
+        DeepTest::RSpecDetector.if_rspec_available do
+          require spec_support_path
+        end
+      end
+
+      def deploy_agents
+        @agents_deployed = true
+        super
+        warlock.exit_when_none_running
+      end
+
+      def agents_deployed?
+        @agents_deployed
+      end
+
+      def forked(*args)
+        $stdout.reopen("/dev/null")
+        $stderr.reopen("/dev/null")
+        super
+      end
+
+      def execute(innie, outie, grace_period)
+        innie.close
+
+        switchboard = Telegraph::Switchboard.new
+        operator = Telegraph::Operator.listen "0.0.0.0", 0, switchboard
+
+        DeepTest.logger.debug { "Beachhead started on port #{operator.port}" }
+
+        outie.write operator.port
+        outie.close
+
+        launch_mercy_killer grace_period
+
+        loop do
+          begin
+            switchboard.process_messages :timeout => 1 do |message, wire|
+              case message.body
+              when LoadFiles
+                load_files message.body.files
+              when DeployAgents
+                deploy_agents
+                wire.send_message Done
+                operator.shutdown
+                break
+              end
+            end
+          end
+        end
+      end
+
+      def daemonize(grace_period = MERCY_KILLING_GRACE_PERIOD)
+        innie, outie = IO.pipe
+
+        warlock.start "Beachhead", self, innie, outie, grace_period
+
+        outie.close
+        port = innie.gets
+        innie.close
+        port.to_i
+      end
+
+      unless defined? DeployAgents
+        DeployAgents = "DeployAgents"
+        Done = "Done"
+        class LoadFiles
+          attr_reader :files
+          def initialize(files)
+            @files = files
+          end
+        end
+      end
+    end
+  end
+end