deep_test 1.1.4 → 1.2.0

data/lib/deep_test/database/mysql_setup_listener.rb

@@ -0,0 +1,109 @@
+module DeepTest
+  module Database
+    #
+    # SetupListener implementation for MySQL.
+    #
+    class MysqlSetupListener < SetupListener
+      class <<self
+        #
+        # ActiveRecord configuration to use when connecting to
+        # MySQL to create databases, drop database, and grant
+        # privileges.  By default, connects to information_schema
+        # on localhost as root with no password.
+        #
+        attr_accessor :admin_configuration
+      end
+      self.admin_configuration = {
+        :adapter  => "mysql",
+        :host     => "localhost",
+        :username => "root",
+        :database => "information_schema"
+      }
+
+      #
+      # Creates database and grants privileges (via +grant_privileges+)
+      # on it via ActiveRecord connection based on admin_configuration.
+      #
+      def create_database
+        admin_connection do |connection|
+          connection.create_database worker_database
+          grant_privileges connection
+        end
+      end
+
+      #
+      # Grants 'all' privilege on worker database to username and password
+      # specified by worker database config.  If your application has
+      # special database privilege needs beyond 'all', you should override
+      # this method and grant them.
+      #
+      def grant_privileges(connection)
+        sql = %{grant all on #{worker_database}.* 
+            to %s@'localhost' identified by %s;} % [
+          connection.quote(worker_database_config[:username]),
+          connection.quote(worker_database_config[:password])
+        ]
+        connection.execute sql
+      end
+
+      #
+      # Drops database via ActiveRecord connection based on admin_configuration
+      #
+      def drop_database
+        admin_connection do |connection|
+          connection.drop_database worker_database
+        end
+      end
+
+      #
+      # Dumps schema from master database using mysqldump command
+      #
+      def dump_schema
+        config = command_line_config(master_database_config)
+        system "mysqldump -R #{config} > #{dump_file_name}"
+        raise "Error Dumping schema" unless $?.success?
+      end
+
+      #
+      # Loads dumpfile into worker database using mysql command
+      #
+      def load_schema
+        config = command_line_config(worker_database_config)
+        system "mysql #{config} < #{dump_file_name}"
+        raise "Error Loading schema" unless $?.success?
+      end
+
+      #
+      # Location to store dumpfile.  The default assumes you are testing
+      # a Rails project.  You should override this if you are not using Rails
+      # or would like the dump file to be something other than the default
+      #
+      def dump_file_name
+        "#{RAILS_ROOT}/db/deep_test_schema.sql"
+      end
+
+      def system(command) # :nodoc:
+        DeepTest.logger.info command
+        super command
+      end
+
+      def command_line_config(config) # :nodoc:
+        command =  ['-u', config[:username]]
+        command += ["-p#{config[:password]}"] if config[:password]
+        command += ['-h', config[:host]] if config[:host]
+        command += ['-P', config[:port]] if config[:port]
+        command += ['-S', config[:socket]] if config[:socket]
+        command += [config[:database]]
+        command.join(' ') 
+      end
+
+      def admin_connection # :nodoc:
+        conn = ActiveRecord::Base.mysql_connection(self.class.admin_configuration)
+        yield conn
+      ensure
+        conn.disconnect! if conn
+      end
+    end
+  end
+end
+

data/lib/deep_test/database/setup_listener.rb

@@ -0,0 +1,116 @@
+module DeepTest
+  module Database
+    #
+    # Skeleton Listener to help with setting up a separate database
+    # for each worker.  Calls +dump_schema+, +load_schema+, +create_database+,
+    # and +drop_database+ hooks provided by subclasses that implement database
+    # setup strategies for particular database flavors.
+    #
+    class SetupListener < NullWorkerListener
+      DUMPED_SCHEMAS = [] unless defined?(DUMPED_SCHEMAS)
+
+      def before_sync # :nodoc:
+        dump_schema_once
+      end
+
+      def before_starting_workers # :nodoc:
+        dump_schema_once
+      end
+
+      def dump_schema_once # :nodoc:
+        schema_name = master_database_config[:database]
+        dump_schema unless DUMPED_SCHEMAS.include?(schema_name)
+        DUMPED_SCHEMAS << schema_name
+      end
+
+      def starting(worker) # :nodoc:
+        @worker = worker
+
+        at_exit do
+          DeepTest.logger.debug("dropping database #{worker_database}")
+          drop_database
+        end
+
+        drop_database
+        create_database
+        connect_to_database
+        load_schema
+      end
+
+      #
+      # Called on each worker after creating database and before loading
+      # schema to initialize connections
+      # 
+      def connect_to_database
+        ActiveRecord::Base.establish_connection(worker_database_config)
+      end
+
+      #
+      # Called in each worker to create the database named by 
+      # +worker_database+.
+      #
+      def create_database
+        raise "Subclass must implement"
+      end
+
+      #
+      # Called in each worker to drop the database created by
+      # +create_database+.  This method is called twice, once before
+      # +create_database+ to ensure that no database exists and once
+      # at exit to clean as the worker process exits.  This method
+      # must not fail if the database does not exist when it is called.
+      #
+      def drop_database
+        raise "Subclass must implement"
+      end
+
+      #
+      # Called before any workers are spawned to dump the schema that
+      # will be used for testing.  When running distributed, this method
+      # is called on the local machine providing the tests to run.
+      #
+      # For distributed testing to work, the schema must be dumped in
+      # location accessible by all worker machines.  The easiest way to
+      # accomplish this is to dump it to a location within the working copy.
+      #
+      def dump_schema
+        raise "Subclass must implement"
+      end
+
+
+      #
+      # Called once in each worker as it is starting to load the schema
+      # dumped from dump_schema.  Subclasses should load the schema definition
+      # into the +worker_database+
+      #
+      def load_schema
+        raise "Subclass must implement"
+      end
+
+      #
+      # ActiveRecord configuration for the worker database.  By default,
+      # the same as +master_database_config+, except that points to
+      # +worker_database+ instead of the database named in the master config.
+      # 
+      def worker_database_config
+        master_database_config.merge(:database => worker_database)
+      end
+
+      #
+      # ActiveRecord configuration for the master database, based on
+      # RAILS_ENV.  If not running Rails, you'll need to override this
+      # to provide the correct configuration.
+      #
+      def master_database_config
+        ActiveRecord::Base.configurations[RAILS_ENV].with_indifferent_access
+      end
+
+      #
+      # Unique name for database on machine that worker is running on.
+      #
+      def worker_database
+        "deep_test_worker_#{@worker.number}_pid_#{Process.pid}" 
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/dispatch_controller.rb

@@ -0,0 +1,53 @@
+module DeepTest
+  module Distributed
+    class DispatchController
+      def initialize(options, receivers)
+        @options = options
+        @receivers = receivers
+      end
+
+      def dispatch(method_name, *args)
+        dispatch_with_options(method_name, {}, *args)
+      end
+
+      def dispatch_with_options(method_name, options, *args)
+        raise NoDispatchReceiversError if @receivers.empty?
+
+        @options.ui_instance.dispatch_starting(method_name)
+
+        threads = @receivers.map do |r|
+          Thread.new do
+            Thread.current[:receiver] = r
+            Timeout.timeout(@options.timeout_in_seconds) do
+              r.send method_name, *args
+            end
+          end
+        end
+
+        results = []
+        threads.each do |t|
+          begin
+            results << t.value
+          rescue Timeout::Error
+            @receivers.delete t[:receiver]
+            DeepTest.logger.error "Timeout dispatching #{method_name} to #{t[:receiver].__drburi}"
+          rescue DRb::DRbConnError
+            @receivers.delete t[:receiver]
+            unless options[:ignore_connection_error]
+              DeepTest.logger.error "Connection Refused dispatching #{method_name} to #{t[:receiver].__drburi}"
+            end
+          rescue Exception => e
+            @receivers.delete t[:receiver]
+            DeepTest.logger.error "Exception while dispatching #{method_name} to #{t[:receiver].__drburi} #{e.message}"
+          end
+        end
+
+        results
+      ensure
+        @options.ui_instance.dispatch_finished(method_name)
+      end
+    end
+
+    class NoDispatchReceiversError < StandardError; end
+  end
+end

data/lib/deep_test/distributed/drb_client_connection_info.rb

@@ -0,0 +1,15 @@
+module DeepTest
+  module Distributed
+    class DRbClientConnectionInfo
+      attr_reader :address
+
+      def initialize
+        info = Thread.current['DRb']
+        raise "No DRb client found" unless info && info['client']
+        peeraddr = info['client'].peeraddr
+        DeepTest.logger.debug("DRbClientConnection info: #{peeraddr.inspect}")
+        @address = peeraddr[3]
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/filename_resolver.rb

@@ -0,0 +1,40 @@
+module DeepTest
+  module Distributed
+    class FilenameResolver
+      def initialize(base_path)
+        @base_path = base_path
+      end
+
+      def resolve(filename)
+        return resolve("/" + filename) unless filename[0] == ?/
+
+        return filename.sub(@cached_replaced_path, @base_path) if @cached_replaced_path
+
+        each_potential_filename(filename) do |potential_filename|
+          if File.exist?(potential_filename)
+            cache_resolution(filename, potential_filename)
+            return potential_filename 
+          end
+        end
+
+        raise "Filename resolution failed.  Cannot resolve #{filename} within #{@base_path}"
+      end
+
+      def cache_resolution(original_filename, resolved_filename)
+        @cached_replaced_path = original_filename.sub(
+           resolved_filename.sub(@base_path, ""), ""
+        )
+      end
+
+      def each_potential_filename(filename)
+        basename = File.basename(filename)
+        dirs = File.dirname(filename).split('/')
+
+        begin
+          dirs.shift
+          yield [@base_path, dirs, basename].flatten.join('/')
+        end until dirs.empty?
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/master_test_server.rb

@@ -0,0 +1,52 @@
+module DeepTest
+  module Distributed
+    class MasterTestServer
+      include ERB::Util
+
+      STATUS_PORT = 4020 unless defined?(STATUS_PORT)
+
+      attr_reader :servers
+
+      def initialize(servers)
+        @servers = servers
+      end
+
+      def show_status(req, res)
+        template = File.read(File.dirname(__FILE__) + "/show_status.rhtml")
+        res.body = ERB.new(template).result(binding)
+      end
+
+      def test_server_statuses
+        @servers.map do |s|
+          status = begin
+                     s.status
+                   rescue Exception => e
+                     e
+                   end
+
+          [s.__drburi, status]
+        end
+      end
+
+      def self.start(uri, server_uris)
+        master = start_drb(uri, server_uris)
+        start_http(master)
+        DeepTest.logger.info "MasterTestServer listening at #{DRb.uri}"
+        DRb.thread.join
+      end
+
+      def self.start_drb(uri, server_uris)
+        servers = server_uris.map {|server_uri| DRbObject.new_with_uri server_uri}
+        master = DeepTest::Distributed::MasterTestServer.new(servers)
+        DRb.start_service(uri, master)
+        master
+      end
+
+      def self.start_http(master)
+        s = WEBrick::HTTPServer.new :Port => STATUS_PORT
+        s.mount_proc("/", &master.method(:show_status))
+        Thread.new {s.start}
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/multi_test_server_proxy.rb

@@ -0,0 +1,44 @@
+module DeepTest
+  module Distributed
+    class MultiTestServerProxy
+      def initialize(options, slaves)
+        DeepTest.logger.debug "MultiTestServerProxy#initialize #{slaves.length} slaves"
+        @slave_controller = DispatchController.new(options, slaves)
+      end
+
+      def spawn_worker_server(options)
+        DeepTest.logger.debug "dispatch spawn_worker_server for #{options.origin_hostname}"
+        WorkerServerProxy.new options,
+                              @slave_controller.dispatch(:spawn_worker_server, 
+                                                         options)
+      end
+
+      def sync(options)
+        DeepTest.logger.debug "dispatch sync for #{options.origin_hostname}"
+        @slave_controller.dispatch(:sync, options)
+      end
+
+      class WorkerServerProxy
+        def initialize(options, slaves)
+          DeepTest.logger.debug "WorkerServerProxy#initialize #{slaves.inspect}"
+          @slave_controller = DispatchController.new(options, slaves)
+        end
+
+        def load_files(files)
+          DeepTest.logger.debug "dispatch load_files"
+          @slave_controller.dispatch(:load_files, files)
+        end
+
+        def start_all
+          DeepTest.logger.debug "dispatch start_all"
+          @slave_controller.dispatch(:start_all)
+        end
+
+        def stop_all
+          DeepTest.logger.debug "dispatch stop_all"
+          @slave_controller.dispatch_with_options(:stop_all, :ignore_connection_error => true)
+        end
+      end
+    end
+  end
+end

data/lib/deep_test/distributed/null_work_unit.rb

@@ -0,0 +1,12 @@
+module DeepTest
+  module Distributed
+    #
+    # Work Unit used to measure throughput of workers.
+    #
+    class NullWorkUnit
+      def run
+        :null_work_unit_result
+      end
+    end
+  end
+end