specjour 0.4.1 → 0.5.0

data/History.markdown

@@ -1,6 +1,41 @@
 History
 =======
 
+0.5.0 / 2012-02-20
+----------------------
+
+* [changed] Printer uses UNIX select instead of GServer (threads)
+* [changed] Database is always dropped and reloaded using schema.rb or
+  structure.sql
+* [removed] RSpec < 2.8 compatibility
+* [added] Memory utilizing forks. No longer forking and execing means workers
+  start running tests faster.
+* [added] Configuration.after_load hook; runs after loading the environment
+* [added] Configurable rsync port
+* [added] Specs distributed by example, not file! Means better
+  distribution/fast spec suites.
+* [added] Rails compiled asset directory (tmp/cache) to the rsync inclusion
+  list. Workers won't have to compile assets during integration tests.
+* [fixed] SQL structure files can be used to build the database.
+* [fixed] Long timeout while waiting for bonjour requests. The bonjour code has
+  been rewritten.
+* [fixed] Load specjour in its own environment when running bundle exec specjour
+* [fixed] Forks running their parent's exit handlers.
+* [fixed] Database creation when the app depends on a database upon environment
+  load (something as simple as a scope would cause this dependency). As long as
+  the regular test environment can be loaded, a worker without a database
+  shouldn't raise an exception, instead the db should be created.
+
+[Full Changelog](https://github.com/sandro/specjour/compare/v0.4.1...0.5.0)
+
+0.4.1 / 2011-06-17
+------------------
+
+l4rk and leshill
+
+* [fixed] Cucumber failure reports not displayed
+
+
 0.4.0 / 2011-03-09
 ------------------
 

data/README.markdown

@@ -102,6 +102,15 @@ By default, the dispatcher looks for managers matching the project's directory n
     ~/bizconf $ specjour listen -p bizconf_09
     ~/bizconf $ specjour -a bizconf_09
 
+## Working with git
+Commit the .specjour directory but ignore the performance file. The performance
+file constantly changes, there's no need to commit it. Specjour uses it in an
+attempt to optimize the run order; ensuring each machine gets at least one
+long-running test.
+
+    $ cat .gitignore
+    /.specjour/performance
+
 ## Compatibility
 
 * RSpec 2

data/Rakefile

@@ -1,5 +1,4 @@
-require 'rubygems'
-require 'rake'
+require 'bundler/gem_tasks'
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)
@@ -19,12 +18,12 @@ end
 
 desc "tag, push gem, push to github"
 task :prerelease do
-  version = `cat VERSION`.strip
+  require 'specjour'
   command = %(
-    git tag v#{version} &&
+    git tag v#{Specjour::VERSION} &&
     rake build &&
     git push &&
-    gem push pkg/specjour-#{version}.gem &&
+    gem push pkg/specjour-#{Specjour::VERSION}.gem &&
     git push --tags
   )
   puts command

data/lib/specjour/cli.rb

@@ -10,25 +10,29 @@ module Specjour
       method_option :alias, :aliases => "-a", :desc => "Project name advertised to listeners"
     end
 
+    def self.rsync_port_option
+      method_option :rsync_port, :type => :numeric, :default => 23456, :desc => "Port to use for rsync daemon"
+    end
+
+    # allow specjour to be called with path arguments
     def self.start(original_args=ARGV, config={})
-      real_tasks = all_tasks.keys | HELP_MAPPINGS
+      real_tasks = all_tasks.keys | @map.keys
       unless real_tasks.include? original_args.first
         original_args.unshift default_task
       end
       super(original_args)
     end
 
-
     default_task :dispatch
 
     class_option :log, :aliases => "-l", :type => :boolean, :desc => "Print debug messages to $stderr"
 
-
-    desc "listen", "Wait for incoming tests"
-    long_desc <<-D
+    desc "listen", "Listen for incoming tests to run"
+    long_desc <<-DESC
       Advertise availability to run tests for the current directory.
-    D
+    DESC
     worker_option
+    rsync_port_option
     method_option :projects, :aliases => "-p", :type => :array, :desc => "Projects supported by this listener"
     def listen
       handle_logging
@@ -38,52 +42,66 @@ module Specjour
       Specjour::Manager.new(args).start
     end
 
-    desc "dispatch [PROJECT_PATH]", "Run tests in the current directory"
+    desc "load", "load the app, then fork workers", :hide => true
+    worker_option
+    method_option :printer_uri, :required => true
+    method_option :project_path, :required => true
+    method_option :task, :required => true
+    method_option :test_paths, :type => :array, :default => []
+    method_option :quiet, :type => :boolean, :default => false
+    def load
+      handle_logging
+      handle_workers
+      append_to_program_name "load"
+      Specjour::Loader.new(args).start
+    end
+
+    desc "dispatch [test_paths]", "Send tests to a listener"
     worker_option
     dispatcher_option
-    def dispatch(path = Dir.pwd)
+    rsync_port_option
+    long_desc <<-DESC
+      This is run when you simply type `specjour`.
+      By default, it will run the specs and features found in the current directory.
+      If you like, you can run a subset of tests by specifying the folder containing the tests.\n
+      Examples\n
+      `specjour dispatch spec`\n
+      `specjour dispatch features`\n
+      `specjour dispatch spec/models features/sign_up.feature`\n
+    DESC
+    def dispatch(*paths)
       handle_logging
       handle_workers
-      handle_dispatcher(path)
+      handle_dispatcher(paths)
       append_to_program_name "dispatch"
       Specjour::Dispatcher.new(args).start
     end
 
-    desc "prepare [PROJECT_PATH]", "Prepare all listening workers"
-    long_desc <<-D
+    desc "prepare [PROJECT_PATH]", "Run the prepare task on all listening workers"
+    long_desc <<-DESC
       Run the Specjour::Configuration.prepare block on all listening workers.
-      Defaults to dropping and schema loading the database.
-    D
+      Defaults to dropping the database, then loading the schema.
+    DESC
     worker_option
     dispatcher_option
+    rsync_port_option
     def prepare(path = Dir.pwd)
       handle_logging
       handle_workers
-      handle_dispatcher(path)
+      args[:project_path] = File.expand_path(path)
+      args[:project_alias] = args.delete(:alias)
+      args[:test_paths] = []
       args[:worker_task] = 'prepare'
       append_to_program_name "prepare"
       Specjour::Dispatcher.new(args).start
     end
 
+    map %w(-v --version) => :version
     desc "version", "Show the current version"
     def version
       puts Specjour::VERSION
     end
 
-    desc "work", "INTERNAL USE ONLY", :hide => true
-    method_option :project_path, :required => true
-    method_option :printer_uri, :required => true
-    method_option :number, :type => :numeric, :required => true
-    method_option :preload_spec
-    method_option :preload_feature
-    method_option :task, :required => true
-    method_option :quiet, :type => :boolean
-    def work
-      handle_logging
-      append_to_program_name "work"
-      Specjour::Worker.new(args).start
-    end
-
     protected
 
     def append_to_program_name(command)
@@ -102,9 +120,15 @@ module Specjour
       args[:worker_size] = options["workers"] || CPU.cores
     end
 
-    def handle_dispatcher(path)
-      args[:project_path] = path
+    def handle_dispatcher(paths)
+      if paths.empty?
+        args[:project_path] = Dir.pwd
+      else
+        args[:project_path] = File.expand_path(paths.first.sub(/(spec|features).*$/, ''))
+      end
+      args[:test_paths] = paths
       args[:project_alias] = args.delete(:alias)
+      raise ArgumentError, "Cannot dispatch line numbers" if paths.any? {|p| p =~ /:\d+/}
     end
   end
 end

data/lib/specjour/configuration.rb

@@ -2,18 +2,24 @@ module Specjour
   module Configuration
     extend self
 
-    attr_writer :before_fork, :after_fork, :prepare
+    attr_writer :before_fork, :after_fork, :after_load, :prepare
 
-    # This block is run by each worker the manager forks.
-    # The Rails plugin uses this block to clear the databases defined in
-    # ActiveRecord.
-    # Set your own block if the default doesn't work for you.
+    # This block is run by each worker before they begin running tests.
+    # The default action is to migrate the database, and clear it of any old
+    # data.
     def after_fork
       @after_fork ||= default_after_fork
     end
 
-    # This block is run after before forking. When ActiveRecord is
-    # defined, the default before_block disconnects from the database.
+    # This block is run after the manager loads the app into memory, but before
+    # forking new worker processes. The default action is to disconnect from
+    # the ActiveRecord database.
+    def after_load
+      @after_load ||= default_after_load
+    end
+
+    # This block is run by the manager before forking workers. The default
+    # action is to run bundle install.
     def before_fork
       @before_fork ||= default_before_fork
     end
@@ -29,6 +35,7 @@ module Specjour
     def reset
       @before_fork = nil
       @after_fork = nil
+      @after_load = nil
       @prepare = nil
     end
 
@@ -40,7 +47,6 @@ module Specjour
 
     def default_before_fork
       lambda do
-        ActiveRecord::Base.remove_connection if defined?(ActiveRecord::Base)
         bundle_install
       end
     end
@@ -51,6 +57,12 @@ module Specjour
       end
     end
 
+    def default_after_load
+      lambda do
+        ActiveRecord::Base.remove_connection if rails_with_ar?
+      end
+    end
+
     def default_prepare
       lambda do
         if rails_with_ar?

data/lib/specjour/connection.rb

@@ -6,7 +6,7 @@ module Specjour
     attr_reader :uri
     attr_writer :socket
 
-    def_delegators :socket, :flush, :closed?, :gets, :each
+    def_delegators :socket, :flush, :close, :closed?, :gets, :each
 
     def self.wrap(established_connection)
       host, port = established_connection.peeraddr.values_at(3,1)
@@ -26,7 +26,7 @@ module Specjour
     end
 
     def disconnect
-      socket.close
+      socket.close unless socket && socket.closed?
     end
 
     def socket
@@ -34,9 +34,8 @@ module Specjour
     end
 
     def timeout(&block)
-      Timeout.timeout(2, &block)
+      Timeout.timeout(0.5, &block)
     rescue Timeout::Error
-      raise Error, "Connection to dispatcher timed out", []
     end
 
     def next_test
@@ -78,7 +77,7 @@ module Specjour
 
     def will_reconnect(&block)
       block.call
-    rescue SystemCallError => error
+    rescue SystemCallError, IOError => error
       unless Specjour.interrupted?
         reconnect
         retry

data/lib/specjour/cpu.rb

@@ -1,7 +1,7 @@
 module Specjour
   module CPU
     def self.cores
-      case RUBY_PLATFORM
+      case platform
       when /darwin/
         command('hostinfo') =~ /^(\d+).+physically/
         $1.to_i
@@ -15,5 +15,9 @@ module Specjour
     def self.command(cmd)
       %x(#{cmd})
     end
+
+    def self.platform
+      RUBY_PLATFORM
+    end
   end
 end

data/lib/specjour/cucumber/preloader.rb

@@ -1,7 +1,8 @@
 module Specjour
   module Cucumber
     module Preloader
-      def self.load(feature_file)
+      def self.load
+        require 'cucumber' unless defined?(::Cucumber::Cli)
         configuration = ::Cucumber::Cli::Configuration.new
         configuration.parse! []
         runtime = ::Cucumber::Runtime.new(configuration)

data/lib/specjour/cucumber.rb

@@ -1,25 +1,16 @@
 module Specjour
   module Cucumber
     begin
-      require 'cucumber'
       require 'cucumber/formatter/progress'
 
       require 'specjour/cucumber/distributed_formatter'
       require 'specjour/cucumber/final_report'
       require 'specjour/cucumber/preloader'
-      require 'specjour/cucumber/main_ext'
       require 'specjour/cucumber/runner'
-
-      ::Cucumber::Cli::Options.class_eval { def print_profile_information; end }
     rescue LoadError
     end
 
     class << self; attr_accessor :runtime; end
 
-    def self.wants_to_quit
-      if defined?(::Cucumber) && ::Cucumber.respond_to?(:wants_to_quit=)
-        ::Cucumber.wants_to_quit = true
-      end
-    end
   end
 end

data/lib/specjour/db_scrub.rb

@@ -1,17 +1,14 @@
+# encoding: utf-8
 module Specjour
   module DbScrub
 
     begin
       require 'rake'
-      if defined?(Rails) && Rails.version =~ /^3/
-        task(:environment) {}
+      extend Rake::DSL if defined?(Rake::DSL)
+      if defined?(Rails)
+        Rake::Task.define_task(:environment) { }
         load 'rails/tasks/misc.rake'
         load 'active_record/railties/databases.rake'
-      else
-        load 'tasks/misc.rake'
-        load 'tasks/databases.rake'
-        Rake::Task["db:structure:dump"].clear
-        Rake::Task["environment"].clear
       end
     rescue LoadError
       Specjour.logger.debug "Failed to load Rails rake tasks"
@@ -25,18 +22,15 @@ module Specjour
 
     def scrub
       connect_to_database
-      if pending_migrations?
-        puts "Migrating schema for database #{ENV['TEST_ENV_NUMBER']}..."
-        schema_load_task.invoke
-      else
-        purge_tables
-      end
+      puts "Resetting database #{ENV['TEST_ENV_NUMBER']}"
+      schema_load_task.invoke
     end
 
     protected
 
     def connect_to_database
       ActiveRecord::Base.remove_connection
+      ActiveRecord::Base.configurations = Rails.application.config.database_configuration
       ActiveRecord::Base.establish_connection
       connection
     rescue # assume the database doesn't exist
@@ -47,20 +41,12 @@ module Specjour
       ActiveRecord::Base.connection
     end
 
-    def purge_tables
-      connection.disable_referential_integrity do
-        tables_to_purge.each do |table|
-          connection.delete "delete from #{table}"
-        end
-      end
-    end
-
     def pending_migrations?
       ActiveRecord::Migrator.new(:up, 'db/migrate').pending_migrations.any?
     end
 
     def schema_load_task
-      Rake::Task[{ :sql  => "db:test:clone_structure", :ruby => "db:test:load" }[ActiveRecord::Base.schema_format]]
+      Rake::Task[{ :sql  => "db:test:load_structure", :ruby => "db:test:load" }[ActiveRecord::Base.schema_format]]
     end
 
     def tables_to_purge

data/lib/specjour/dispatcher.rb

@@ -4,18 +4,19 @@ module Specjour
     Thread.abort_on_exception = true
     include SocketHelper
 
-    attr_reader :project_alias, :managers, :manager_threads, :hosts, :options, :all_tests, :drb_connection_errors
+    attr_reader :project_alias, :managers, :manager_threads, :hosts, :options, :drb_connection_errors, :test_paths, :rsync_port
     attr_accessor :worker_size, :project_path
 
     def initialize(options = {})
       Specjour.load_custom_hooks
       @options = options
-      @project_path = File.expand_path options[:project_path]
+      @project_path = options[:project_path]
+      @test_paths = options[:test_paths]
       @worker_size = 0
       @managers = []
       @drb_connection_errors = Hash.new(0)
-      find_tests
-      clear_manager_threads
+      @rsync_port = options[:rsync_port]
+      reset_manager_threads
     end
 
     def start
@@ -23,41 +24,23 @@ module Specjour
       gather_managers
       rsync_daemon.start
       dispatch_work
-      printer.join if dispatching_tests?
+      printer.start if dispatching_tests?
       wait_on_managers
       exit printer.exit_status
     end
 
     protected
 
-    def find_tests
-      if project_path.match(/(.+)\/((spec|features)(?:\/\w+)*)$/)
-        self.project_path = $1
-        @all_tests = $3 == 'spec' ? all_specs($2) : all_features($2)
-      else
-        @all_tests = all_specs | all_features
-      end
-    end
-
-    def all_specs(tests_path = 'spec')
-      Dir[File.join(".", tests_path, "**/*_spec.rb")].sort
-    end
-
-    def all_features(tests_path = 'features')
-      Dir[File.join(".", tests_path, "**/*.feature")].sort
-    end
-
     def add_manager(manager)
       set_up_manager(manager)
       managers << manager
       self.worker_size += manager.worker_size
     end
 
-    def command_managers(async = false, &block)
+    def command_managers(&block)
       managers.each do |manager|
         manager_threads << Thread.new(manager, &block)
       end
-      wait_on_managers unless async
     end
 
     def dispatcher_uri
@@ -69,8 +52,7 @@ module Specjour
       managers.each do |manager|
         puts "#{manager.hostname} (#{manager.worker_size})"
       end
-      printer.worker_size = worker_size
-      command_managers(true) { |m| m.dispatch rescue DRb::DRbConnError }
+      command_managers { |m| m.dispatch rescue DRb::DRbConnError }
     end
 
     def dispatching_tests?
@@ -85,12 +67,12 @@ module Specjour
     rescue DRb::DRbConnError => e
       drb_connection_errors[uri] += 1
       Specjour.logger.debug "#{e.message}: couldn't connect to manager at #{uri}"
-      retry if drb_connection_errors[uri] < 5
+      sleep(0.1) && retry if drb_connection_errors[uri] < 5
     end
 
     def fork_local_manager
       puts "No listeners found on this machine, starting one..."
-      manager_options = {:worker_size => options[:worker_size], :registered_projects => [project_alias]}
+      manager_options = {:worker_size => options[:worker_size], :registered_projects => [project_alias], :rsync_port => rsync_port}
       manager = Manager.start_quietly manager_options
       fetch_manager(manager.drb_uri)
       at_exit do
@@ -101,23 +83,23 @@ module Specjour
     end
 
     def gather_managers
-      puts "Looking for managers..."
+      puts "Looking for listeners..."
       gather_remote_managers
       fork_local_manager if local_manager_needed?
-      abort "No managers found" if managers.size.zero?
+      abort "No listeners found" if managers.size.zero?
     end
 
     def gather_remote_managers
-      browser = DNSSD::Service.new
-      Timeout.timeout(3) do
-        browser.browse '_druby._tcp' do |reply|
-          if reply.flags.add?
-            resolve_reply(reply)
-          end
-          browser.stop unless reply.flags.more_coming?
+      replies = []
+      Timeout.timeout(1) do
+        DNSSD.browse!('_druby._tcp') do |reply|
+          replies << reply if reply.flags.add?
+          break unless reply.flags.more_coming?
         end
+        raise Timeout::Error
       end
       rescue Timeout::Error
+        replies.each {|r| resolve_reply(r)}
     end
 
     def local_manager_needed?
@@ -129,7 +111,7 @@ module Specjour
     end
 
     def printer
-      @printer ||= Printer.start(all_tests)
+      @printer ||= Printer.new
     end
 
     def project_alias
@@ -140,34 +122,36 @@ module Specjour
       @project_name ||= File.basename(project_path)
     end
 
-    def clear_manager_threads
+    def reset_manager_threads
       @manager_threads = []
     end
 
     def resolve_reply(reply)
-      DNSSD.resolve!(reply) do |resolved|
+      DNSSD.resolve!(reply.name, reply.type, reply.domain, flags=0, reply.interface) do |resolved|
         Specjour.logger.debug "Bonjour discovered #{resolved.target}"
-        resolved_ip = ip_from_hostname(resolved.target)
-        uri = URI::Generic.build :scheme => reply.service_name, :host => resolved_ip, :port => resolved.port
-        fetch_manager(uri)
-        resolved.service.stop if resolved.service.started?
+        if resolved.text_record && resolved.text_record['version'] == Specjour::VERSION
+          resolved_ip = ip_from_hostname(resolved.target)
+          uri = URI::Generic.build :scheme => reply.service_name, :host => resolved_ip, :port => resolved.port
+          fetch_manager(uri)
+        else
+          puts "Found #{resolved.target} but its version doesn't match v#{Specjour::VERSION}. Skipping..."
+        end
+        break unless resolved.flags.more_coming?
       end
     end
 
     def rsync_daemon
-      @rsync_daemon ||= RsyncDaemon.new(project_path, project_name)
+      @rsync_daemon ||= RsyncDaemon.new(project_path, project_name, rsync_port)
     end
 
     def set_up_manager(manager)
       manager.project_name = project_name
       manager.dispatcher_uri = dispatcher_uri
-      manager.preload_spec = all_tests.detect {|f| f =~ /_spec\.rb$/}
-      manager.preload_feature = all_tests.detect {|f| f =~ /\.feature$/}
+      manager.test_paths = test_paths
       manager.worker_task = worker_task
       at_exit do
         begin
           manager.interrupted = Specjour.interrupted?
-          manager.kill_worker_processes
         rescue DRb::DRbConnError
         end
       end
@@ -175,7 +159,7 @@ module Specjour
 
     def wait_on_managers
       manager_threads.each {|t| t.join; t.exit}
-      clear_manager_threads
+      reset_manager_threads
     end
 
     def worker_task

data/lib/specjour/fork.rb

@@ -0,0 +1,26 @@
+module Specjour::Fork
+
+  module_function
+
+  # fork, but don't run the parent's exit handlers
+  # The one exit handler we lose however, is the printing out
+  # of exceptions, so reincorporate that.
+  def fork
+    Kernel.fork do
+      at_exit { exit! }
+      begin
+        yield
+      rescue StandardError => e
+        $stderr.puts "#{e.class} #{e.message}", e.backtrace
+      end
+    end
+  end
+
+  def fork_quietly
+    fork do
+      $stdout = StringIO.new
+      yield
+    end
+  end
+
+end