jstorimer-deep-test 0.1.0

data/infrastructure/timewarp/test/timewarp_test.rb

@@ -0,0 +1,45 @@
+$LOAD_PATH << File.dirname(__FILE__) + "/../lib"
+require 'test/unit'
+require 'timewarp'
+require 'rubygems'
+require 'shoulda'
+
+class TimewarpTest < Test::Unit::TestCase
+  context "control_timeline" do
+    should "yield" do
+      assert_equal :foo, Timewarp.control_timeline(proc {}) {:foo}
+    end
+    
+    should "repeatedly call supplied proc every time Time.now is called" do
+      time_values = [:timepoint_1, :timepoint_2]
+      Timewarp.control_timeline(proc {time_values.shift}) do
+        assert_equal :timepoint_1, Time.now
+        assert_equal :timepoint_2, Time.now
+        assert_equal nil, Time.now
+      end
+    end
+
+    should "restorte the normal flow of time even if exception occurs" do
+      Timewarp.control_timeline(proc {}) {raise "error"} rescue nil
+      time_1 = Time.now
+      sleep 0.01
+      time_2 = Time.now
+
+      assert time_1 < time_2
+    end
+  end
+
+  context "freeze" do
+    should "yield" do
+      assert_equal :foo, Timewarp.freeze(:frozen_time) {:foo}
+    end
+
+    should "return a constant time" do
+      assert_equal :frozen_time, Timewarp.freeze(:frozen_time) {Time.now}
+    end
+
+    should "parse the time if it is a string" do
+      assert_equal Time.parse("2009-07-01 12:01:01"), Timewarp.freeze("2009-07-01 12:01:01") {Time.now}
+    end
+  end
+end

data/lib/deep_test/agent.rb

@@ -0,0 +1,108 @@
+module DeepTest
+  class Agent
+    include Demon
+    attr_reader :number
+
+    def initialize(number, options, listener)
+      @number = number
+      @listener = listener
+      @options = options
+    end
+
+    def connect(stream_to_parent_process)
+      DeepTest.logger.debug { "Agent: Connecting to #{@options.origin_hostname}:#{@options.server_port}" }
+      @options.connect_to_central_command do |wire|
+        stream_to_parent_process.puts "Connected"
+        stream_to_parent_process.close rescue nil
+        yield wire
+      end
+    ensure
+      stream_to_parent_process.close unless stream_to_parent_process.closed?
+    end
+
+    def execute(stream_from_child_process, stream_to_parent_process)
+      stream_from_child_process.close
+      connect(stream_to_parent_process) do |wire|
+        reseed_random_numbers
+        reconnect_to_database
+
+        @listener.starting(self)
+        wire.send_message CentralCommand::NeedWork
+
+        while work_unit_message = next_work_unit_message(wire)
+          @listener.starting_work(self, work_unit_message.body)
+
+          result = begin
+                     Metrics::Measurement.send_home("Agents Performing Work", wire, @options) do
+                       work_unit_message.body.run
+                     end
+                   rescue Exception => error
+                     Error.new(work_unit_message.body, error)
+                   end
+
+          @listener.finished_work(self, work_unit_message.body, result)
+          send_result wire, work_unit_message, result
+        end
+      end
+    rescue CentralCommand::NoWorkUnitsRemainingError
+      DeepTest.logger.debug { "Agent #{number}: no more work to do" }
+    end
+
+    def next_work_unit_message(wire)
+      Metrics::Measurement.send_home("Agents Retrieving Work", wire, @options) do
+        begin
+          message = wire.next_message(:timeout => 2)
+          next message.body == CentralCommand::NoMoreWork ? nil : message
+        rescue Telegraph::NoMessageAvailable
+          DeepTest.logger.debug { "Agent: NoMessageAvailable" }
+          retry
+        rescue Telegraph::LineDead
+          DeepTest.logger.debug { "Agent: LineDead" }
+          next nil
+        end
+      end
+    end
+
+    def send_result(wire, work_unit_message, result)
+      wire.send_message result, :ack => work_unit_message
+    end
+
+    def reconnect_to_database
+      ActiveRecord::Base.connection.reconnect! if defined?(ActiveRecord::Base)
+    end
+
+    def reseed_random_numbers
+      srand
+    end
+
+
+    class Error
+      include CentralCommand::Result
+
+      attr_accessor :work_unit, :error
+
+      def initialize(work_unit, error)
+        @work_unit, @error = work_unit, error
+      end
+
+      def _dump(limit)
+        Marshal.dump([@work_unit, @error], limit)
+      rescue
+        Marshal.dump(["<< Undumpable >>", @error], limit)
+      end
+
+      def self._load(string)
+        new *Marshal.load(string)
+      end
+
+      def ==(other)
+        work_unit == other.work_unit &&
+            error == other.error
+      end
+
+      def to_s
+        "#{@work_unit}: #{@error}\n" + (@error.backtrace || []).join("\n")
+      end
+    end
+  end
+end

data/lib/deep_test/central_command.rb

@@ -0,0 +1,165 @@
+require 'set'
+
+module DeepTest
+  class CentralCommand
+    attr_reader :operator
+    attr_reader :switchboard
+    attr_reader :data
+
+    def initialize(options)
+      @options = options
+      @work_queue = Queue.new
+      @results_mutex = Mutex.new
+      @results_condvar = ConditionVariable.new
+      @results = []
+      @data = Metrics::Data.new
+    end
+
+    def done_with_work
+      @done_with_work = true
+    end
+
+    def take_result
+      @results_mutex.synchronize do
+        loop do
+          if @results.any?
+            return @results.shift
+          else
+            @results_condvar.wait @results_mutex
+            raise NoAgentsRunningError unless @results.any? || @switchboard.any_live_wires?
+          end
+        end
+      end
+    end
+
+    def take_work
+      @work_queue.pop(true)
+    rescue ThreadError => e
+      if e.message == "queue empty"
+        raise NoWorkUnitsRemainingError if @done_with_work
+        raise NoWorkUnitsAvailableError
+      else
+        raise
+      end
+    end
+
+    def write_result(result)
+      @results_mutex.synchronize do
+        @results << result
+        @results_condvar.signal
+      end
+    end
+
+    def write_work(work_unit)
+      @work_queue.push work_unit
+    end
+
+    def stdout
+      $stdout
+    end
+
+    def stderr
+      $stderr
+    end
+
+    def self.start(options)
+      central_command = new(options)
+      central_command.start
+      central_command
+    end
+
+    def start
+      @switchboard = Telegraph::Switchboard.new
+      @operator = Telegraph::Operator.listen("0.0.0.0", 0, @switchboard)
+      @options.server_port = @operator.port
+      @process_messages_thread = Thread.new { process_messages }
+
+      DeepTest.logger.info { "Started DeepTest service on port #{@operator.port}" }
+    end
+
+    unless defined?(NeedWork)
+      NeedWork = "NeedWork" 
+      NoMoreWork = "NoMoreWork"
+      module Result; end
+      module Operation; end
+    end
+
+    def process_messages
+      loop do
+        begin
+          return if @stop_process_messages
+          wires_waiting_for_work.each { |w| send_work w }
+
+          @results_mutex.synchronize do
+            # make take_result wake up and check if any agents are running
+            @results_condvar.signal
+          end
+
+          message, wire = switchboard.next_message(:timeout => 0.5)
+
+          case message.body
+          when NeedWork
+            send_work wire
+          when Result 
+            write_result message.body
+            send_work wire
+          when Operation
+            message.body.execute
+          when Metrics::Measurement
+            data.add message.body
+          else 
+            raise UnexpectedMessageError, message.inspect
+          end
+
+        rescue Telegraph::NoMessageAvailable
+          retry
+        rescue Exception => e
+          raise unless @stop_process_messages
+        end
+      end
+    end
+
+    def wires_waiting_for_work
+      @wires_waiting_for_work ||= Set.new
+    end
+
+    def send_work(wire)
+      begin
+        wire.send_message take_work, :need_ack => true
+        wires_waiting_for_work.delete wire
+
+      rescue NoWorkUnitsAvailableError
+        put_abandonded_work_back_on_the_queue
+        wires_waiting_for_work.add wire
+
+      rescue NoWorkUnitsRemainingError
+        wire.send_message NoMoreWork
+        wires_waiting_for_work.delete wire
+
+      end
+    end
+
+    def put_abandonded_work_back_on_the_queue
+      closed_wires = switchboard.using_wires { |wires| wires.select {|w| w.closed? }}
+      closed_wires.each do |wire|
+        wire.unacked_messages.each do |m|
+          write_work m.body
+          switchboard.drop_wire wire
+        end
+      end
+    end
+
+    def stop
+      @stop_process_messages = true
+      operator.shutdown
+      @process_messages_thread.join
+      data.save @options.metrics_file if @options.gathering_metrics?
+    end
+
+    class NoWorkUnitsAvailableError < StandardError; end
+    class NoWorkUnitsRemainingError < StandardError; end
+    class NoAgentsRunningError < StandardError; end
+    class CheckIfAgentsAreStillRunning < StandardError; end
+    class UnexpectedMessageError < StandardError; end
+  end
+end

data/lib/deep_test/cpu_info.rb

@@ -0,0 +1,22 @@
+module DeepTest
+  class CpuInfo
+    attr_accessor :platform
+
+    def initialize(platform = RUBY_PLATFORM)
+      @platform = platform
+    end
+
+    def count
+      case platform
+      when /darwin/
+        output = `sysctl -n hw.ncpu`
+        output.strip.to_i
+      when /linux/
+        File.readlines("/proc/cpuinfo").inject(0) do |count, line|
+          next count + 1 if line =~ /processor\s*:\s*\d+/
+          count
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/database/mysql_setup_listener.rb

@@ -0,0 +1,112 @@
+module DeepTest
+  module Database
+    #
+    # SetupListener implementation for MySQL.
+    #
+    class MysqlSetupListener < SetupListener
+      class <<self
+        #
+        # ActiveRecord configuration to use when connecting to
+        # MySQL to create databases, drop database, and grant
+        # privileges.  By default, connects to information_schema
+        # on localhost as root with no password.
+        #
+        attr_accessor :admin_configuration
+      end
+      self.admin_configuration = {
+        :adapter  => "mysql",
+        :host     => "localhost",
+        :username => "root",
+        :database => "information_schema"
+      }
+
+      #
+      # Creates database and grants privileges (via +grant_privileges+)
+      # on it via ActiveRecord connection based on admin_configuration.
+      #
+      def create_database
+        admin_connection do |connection|
+          connection.create_database agent_database
+          grant_privileges connection
+        end
+      end
+
+      #
+      # Grants 'all' privilege on agent database to username and password
+      # specified by agent database config.  If your application has
+      # special database privilege needs beyond 'all', you should override
+      # this method and grant them.
+      #
+      def grant_privileges(connection)
+        identified_by = if agent_database_config[:password]
+                          %{identified by %s} % connection.quote(agent_database_config[:password])
+                        else
+                          ""
+                        end
+        sql = %{grant all on #{agent_database}.* to %s@'localhost' #{identified_by} ; } % 
+          connection.quote(agent_database_config[:username])
+
+        connection.execute sql
+      end
+
+      #
+      # Drops database via ActiveRecord connection based on admin_configuration
+      #
+      def drop_database
+        admin_connection do |connection|
+          connection.drop_database agent_database
+        end
+      end
+
+      #
+      # Dumps schema from master database using mysqldump command
+      #
+      def dump_schema
+        config = command_line_config(master_database_config)
+        system "mysqldump -R #{config} > #{dump_file_name}"
+        raise "Error Dumping schema" unless $?.success?
+      end
+
+      #
+      # Loads dumpfile into agent database using mysql command
+      #
+      def load_schema
+        config = command_line_config(agent_database_config)
+        system "mysql #{config} < #{dump_file_name}"
+        raise "Error Loading schema" unless $?.success?
+      end
+
+      #
+      # Location to store dumpfile.  The default assumes you are testing
+      # a Rails project.  You should override this if you are not using Rails
+      # or would like the dump file to be something other than the default
+      #
+      def dump_file_name
+        "#{RAILS_ROOT}/db/deep_test_schema.sql"
+      end
+
+      def system(command) # :nodoc:
+        DeepTest.logger.info { command }
+        super command
+      end
+
+      def command_line_config(config) # :nodoc:
+        command =  ['-u', config[:username]]
+        command += ["-p#{config[:password]}"] if config[:password]
+        command += ['-h', config[:host]] if config[:host]
+        command += ['-P', config[:port]] if config[:port]
+        command += ['-S', config[:socket]] if config[:socket]
+        command += [config[:database]]
+        command.join(' ') 
+      end
+
+      def admin_connection # :nodoc:
+        conn = ActiveRecord::Base.mysql_connection(self.class.admin_configuration)
+        yield conn
+      ensure
+        conn.disconnect! if conn
+      end
+    end
+  end
+end
+

data/lib/deep_test/database/setup_listener.rb

@@ -0,0 +1,116 @@
+module DeepTest
+  module Database
+    #
+    # Skeleton Listener to help with setting up a separate database
+    # for each agent.  Calls +dump_schema+, +load_schema+, +create_database+,
+    # and +drop_database+ hooks provided by subclasses that implement database
+    # setup strategies for particular database flavors.
+    #
+    class SetupListener < NullListener
+      DUMPED_SCHEMAS = [] unless defined?(DUMPED_SCHEMAS)
+
+      def before_sync # :nodoc:
+        dump_schema_once
+      end
+
+      def before_starting_agents # :nodoc:
+        dump_schema_once
+      end
+
+      def dump_schema_once # :nodoc:
+        schema_name = master_database_config[:database]
+        dump_schema unless DUMPED_SCHEMAS.include?(schema_name)
+        DUMPED_SCHEMAS << schema_name
+      end
+
+      def starting(agent) # :nodoc:
+        @agent = agent
+
+        at_exit do
+          DeepTest.logger.debug { "dropping database #{agent_database}" }
+          drop_database
+        end
+
+        drop_database
+        create_database
+        connect_to_database
+        load_schema
+      end
+
+      #
+      # Called on each agent after creating database and before loading
+      # schema to initialize connections
+      # 
+      def connect_to_database
+        ActiveRecord::Base.establish_connection(agent_database_config)
+      end
+
+      #
+      # Called in each agent to create the database named by 
+      # +agent_database+.
+      #
+      def create_database
+        raise "Subclass must implement"
+      end
+
+      #
+      # Called in each agent to drop the database created by
+      # +create_database+.  This method is called twice, once before
+      # +create_database+ to ensure that no database exists and once
+      # at exit to clean as the agent process exits.  This method
+      # must not fail if the database does not exist when it is called.
+      #
+      def drop_database
+        raise "Subclass must implement"
+      end
+
+      #
+      # Called before any agents are spawned to dump the schema that
+      # will be used for testing.  When running distributed, this method
+      # is called on the local machine providing the tests to run.
+      #
+      # For distributed testing to work, the schema must be dumped in
+      # location accessible by all agent machines.  The easiest way to
+      # accomplish this is to dump it to a location within the working copy.
+      #
+      def dump_schema
+        raise "Subclass must implement"
+      end
+
+
+      #
+      # Called once in each agent as it is starting to load the schema
+      # dumped from dump_schema.  Subclasses should load the schema definition
+      # into the +agent_database+
+      #
+      def load_schema
+        raise "Subclass must implement"
+      end
+
+      #
+      # ActiveRecord configuration for the agent database.  By default,
+      # the same as +master_database_config+, except that points to
+      # +agent_database+ instead of the database named in the master config.
+      # 
+      def agent_database_config
+        master_database_config.merge(:database => agent_database)
+      end
+
+      #
+      # ActiveRecord configuration for the master database, based on
+      # RAILS_ENV.  If not running Rails, you'll need to override this
+      # to provide the correct configuration.
+      #
+      def master_database_config
+        ActiveRecord::Base.configurations[RAILS_ENV].with_indifferent_access
+      end
+
+      #
+      # Unique name for database on machine that agent is running on.
+      #
+      def agent_database
+        "deep_test_agent_#{@agent.number}_pid_#{Process.pid}" 
+      end
+    end
+  end
+end

data/lib/deep_test/deadlock_detector.rb

@@ -0,0 +1,7 @@
+module DeepTest
+  class DeadlockDetector
+    def self.due_to_deadlock?(error)
+      error && !error.message.to_s.match(/Deadlock found when trying to get lock/).nil?
+    end
+  end
+end

data/lib/deep_test/demon.rb

@@ -0,0 +1,25 @@
+module DeepTest
+  module Demon
+    def forked(name, options, demon_args)
+      options.connect_to_central_command do |wire|
+        ProxyIO.replace_stdout_stderr!(wire) do
+          begin
+            catch(:exit_demon) do
+              Signal.trap("TERM") { throw :exit_demon }
+              execute *demon_args
+            end
+          rescue SystemExit => e
+            raise
+          rescue Exception => e
+            FailureMessage.show self.class.name, "Process #{Process.pid} exiting with excetion: #{e.class}: #{e.message}"
+            raise
+          end
+        end
+      end
+    end
+
+    def execute(*args)
+      raise "#{self.class} must implement the execute method to be a Demon"
+    end
+  end
+end

data/lib/deep_test/distributed/beachhead.rb

@@ -0,0 +1,104 @@
+module DeepTest
+  module Distributed
+    class Beachhead < LocalDeployment
+      include Demon
+
+      MERCY_KILLING_GRACE_PERIOD = 10 * 60 unless defined?(MERCY_KILLING_GRACE_PERIOD)
+
+      def initialize(base_path, options)
+        super options
+        @base_path = base_path
+      end
+
+      def launch_mercy_killer(grace_period)
+        Thread.new do
+          sleep grace_period
+          exit(0) unless agents_deployed?
+        end
+      end
+
+      def load_files(files)
+        spec_support_path = File.expand_path(File.dirname(__FILE__) + "/../spec") 
+        Dir.chdir @base_path
+        resolver = FilenameResolver.new(@base_path)
+        files.each do |file|
+          load resolver.resolve(file)
+        end
+
+        # Load rspec support if rspec is available now that we've loaded the host project files
+        #
+        DeepTest::RSpecDetector.if_rspec_available do
+          require spec_support_path
+        end
+      end
+
+      def deploy_agents
+        @agents_deployed = true
+        super
+        warlock.exit_when_none_running
+      end
+
+      def agents_deployed?
+        @agents_deployed
+      end
+
+      def forked(*args)
+        $stdout.reopen("/dev/null")
+        $stderr.reopen("/dev/null")
+        super
+      end
+
+      def execute(innie, outie, grace_period)
+        innie.close
+
+        switchboard = Telegraph::Switchboard.new
+        operator = Telegraph::Operator.listen "0.0.0.0", 0, switchboard
+
+        DeepTest.logger.debug { "Beachhead started on port #{operator.port}" }
+
+        outie.write operator.port
+        outie.close
+
+        launch_mercy_killer grace_period
+
+        loop do
+          begin
+            switchboard.process_messages :timeout => 1 do |message, wire|
+              case message.body
+              when LoadFiles
+                load_files message.body.files
+              when DeployAgents
+                deploy_agents
+                wire.send_message Done
+                operator.shutdown
+                break
+              end
+            end
+          end
+        end
+      end
+
+      def daemonize(grace_period = MERCY_KILLING_GRACE_PERIOD)
+        innie, outie = IO.pipe
+
+        warlock.start "Beachhead", self, innie, outie, grace_period
+
+        outie.close
+        port = innie.gets
+        innie.close
+        port.to_i
+      end
+
+      unless defined? DeployAgents
+        DeployAgents = "DeployAgents"
+        Done = "Done"
+        class LoadFiles
+          attr_reader :files
+          def initialize(files)
+            @files = files
+          end
+        end
+      end
+    end
+  end
+end