friendlyfashion-parallel_tests 0.9.0

data/lib/parallel_tests/rspec/summary_logger.rb

@@ -0,0 +1,19 @@
+require 'parallel_tests/rspec/failures_logger'
+
+class ParallelTests::RSpec::SummaryLogger < ParallelTests::RSpec::LoggerBase
+  # RSpec 1: dumps 1 failed spec
+  def dump_failure(*args)
+    lock_output do
+      super
+    end
+    @output.flush
+  end
+
+  # RSpec 2: dumps all failed specs
+  def dump_failures(*args)
+    lock_output do
+      super
+    end
+    @output.flush
+  end
+end

data/lib/parallel_tests/tasks.rb

@@ -0,0 +1,134 @@
+require 'rake'
+
+module ParallelTests
+  module Tasks
+    class << self
+      def rails_env
+        ENV['RAILS_ENV'] || 'test'
+      end
+
+      def run_in_parallel(cmd, options={})
+        count = " -n #{options[:count]}" if options[:count]
+        executable = File.expand_path("../../../bin/parallel_test", __FILE__)
+        command = "#{executable} --exec '#{cmd}'#{count}#{' --non-parallel' if options[:non_parallel]}"
+        command << " --advance-number #{options[:advance_number]}" if options[:advance_number]
+        abort unless system(command)
+      end
+
+      # this is a crazy-complex solution for a very simple problem:
+      # removing certain lines from the output without chaning the exit-status
+      # normally I'd not do this, but it has been lots of fun and a great learning experience :)
+      #
+      # - sed does not support | without -r
+      # - grep changes 0 exitstatus to 1 if nothing matches
+      # - sed changes 1 exitstatus to 0
+      # - pipefail makes pipe fail with exitstatus of first failed command
+      # - pipefail is not supported in (zsh)
+      # - defining a new rake task like silence_schema would force users to load parallel_tests in test env
+      # - do not use ' since run_in_parallel uses them to quote stuff
+      # - simple system "set -o pipefail" returns nil even though set -o pipefail exists with 0
+      def suppress_output(command, ignore_regex)
+        activate_pipefail = "set -o pipefail"
+        remove_ignored_lines = %Q{(grep -v "#{ignore_regex}" || test 1)}
+
+        if system("#{activate_pipefail} && test 1")
+          "#{activate_pipefail} && (#{command}) | #{remove_ignored_lines}"
+        else
+          command
+        end
+      end
+
+      def check_for_pending_migrations
+        abort_migrations = "db:abort_if_pending_migrations"
+        if Rake::Task.task_defined?(abort_migrations)
+          Rake::Task[abort_migrations].invoke
+        end
+      end
+
+      # parallel:spec[:count, :pattern, :options]
+      def parse_args(args)
+        # order as given by user
+        args = [args[:count], args[:pattern], args[:options]]
+
+        # count given or empty ?
+        # parallel:spec[2,models,options]
+        # parallel:spec[,models,options]
+        count = args.shift if args.first.to_s =~ /^\d*$/
+        num_processes = count.to_i unless count.to_s.empty?
+        pattern = args.shift
+        options = args.shift
+
+        [num_processes, pattern.to_s, options.to_s]
+      end
+    end
+  end
+end
+
+namespace :parallel do
+  desc "create test databases via db:create --> parallel:create[num_cpus]"
+  task :create, :count do |t,args|
+    ParallelTests::Tasks.run_in_parallel("rake db:create RAILS_ENV=#{ParallelTests::Tasks.rails_env}", args)
+  end
+
+  desc "drop test databases via db:drop --> parallel:drop[num_cpus]"
+  task :drop, :count do |t,args|
+    ParallelTests::Tasks.run_in_parallel("rake db:drop RAILS_ENV=#{ParallelTests::Tasks.rails_env}", args)
+  end
+
+  desc "update test databases by dumping and loading --> parallel:prepare[num_cpus]"
+  task(:prepare, [:count]) do |t,args|
+    ParallelTests::Tasks.check_for_pending_migrations
+    if defined?(ActiveRecord) && ActiveRecord::Base.schema_format == :ruby
+      # dump then load in parallel
+      Rake::Task['db:schema:dump'].invoke
+      Rake::Task['parallel:load_schema'].invoke(args[:count])
+    else
+      # there is no separate dump / load for schema_format :sql -> do it safe and slow
+      args = args.to_hash.merge(:non_parallel => true) # normal merge returns nil
+      ParallelTests::Tasks.run_in_parallel('rake db:test:prepare --trace', args)
+    end
+  end
+
+  # when dumping/resetting takes too long
+  desc "update test databases via db:migrate --> parallel:migrate[num_cpus]"
+  task :migrate, :count do |t,args|
+    ParallelTests::Tasks.run_in_parallel("rake db:migrate RAILS_ENV=#{ParallelTests::Tasks.rails_env}", args)
+  end
+
+  # just load the schema (good for integration server <-> no development db)
+  desc "load dumped schema for test databases via db:schema:load --> parallel:load_schema[num_cpus]"
+  task :load_schema, :count do |t,args|
+    command = "rake db:schema:load RAILS_ENV=#{ParallelTests::Tasks.rails_env}"
+    ParallelTests::Tasks.run_in_parallel(ParallelTests::Tasks.suppress_output(command, "^   ->\\|^-- "), args)
+  end
+
+  desc "load the seed data from db/seeds.rb via db:seed --> parallel:seed[num_cpus]"
+  task :seed, :count do |t,args|
+    ParallelTests::Tasks.run_in_parallel("rake db:seed RAILS_ENV=#{ParallelTests::Tasks.rails_env}", args)
+  end
+
+  ['test', 'spec', 'features'].each do |type|
+    desc "run #{type} in parallel with parallel:#{type}[num_cpus]"
+    task type, [:count, :pattern, :options] do |t, args|
+      ParallelTests::Tasks.check_for_pending_migrations
+
+      $LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), '..'))
+      require "parallel_tests"
+
+      count, pattern, options = ParallelTests::Tasks.parse_args(args)
+      test_framework = {
+        'spec' => 'rspec',
+        'test' => 'test',
+        'features' => 'cucumber'
+      }[type]
+
+      executable = File.join(File.dirname(__FILE__), '..', '..', 'bin', 'parallel_test')
+      command = "#{executable} #{type} --type #{test_framework} " \
+        "-n #{count} "                     \
+        "--pattern '#{pattern}' "          \
+        "--test-options '#{options}'"
+
+      abort unless system(command) # allow to chain tasks e.g. rake parallel:spec parallel:features
+    end
+  end
+end

data/lib/parallel_tests/test/runner.rb

@@ -0,0 +1,134 @@
+module ParallelTests
+  module Test
+    class Runner
+      # --- usually overwritten by other runners
+
+      def self.runtime_log
+        'tmp/parallel_runtime_test.log'
+      end
+
+      def self.test_suffix
+        "_test.rb"
+      end
+
+      def self.test_file_name
+        "test"
+      end
+
+      def self.run_tests(test_files, process_number, options)
+        require_list = test_files.map { |filename| %{"#{File.expand_path filename}"} }.join(",")
+        cmd = "ruby -Itest -e '[#{require_list}].each {|f| require f }' -- #{options[:test_options]}"
+        execute_command(cmd, process_number, options)
+      end
+
+      def self.line_is_result?(line)
+        line =~ /\d+ failure/
+      end
+
+      # --- usually used by other runners
+
+      # finds all tests and partitions them into groups
+      def self.tests_in_groups(tests, num_groups, options={})
+        tests = find_tests(tests, options)
+
+        tests = if options[:group_by] == :found
+          tests.map { |t| [t, 1] }
+        else
+          with_runtime_info(tests)
+        end
+        Grouper.in_even_groups_by_size(tests, num_groups, options)
+      end
+
+      def self.execute_command(cmd, process_number, options)
+        cmd = "TEST_ENV_NUMBER=#{test_env_number(process_number, options)} ; export TEST_ENV_NUMBER; #{cmd}"
+        f = open("|#{cmd}", 'r')
+        output = fetch_output(f)
+        f.close
+        {:stdout => output, :exit_status => $?.exitstatus}
+      end
+
+      def self.find_results(test_output)
+        test_output.split("\n").map {|line|
+          line = line.gsub(/\.|F|\*/,'').gsub(/\e\[\d+m/,'')
+          next unless line_is_result?(line)
+          line
+        }.compact
+      end
+
+      def self.test_env_number(process_number, options)
+        n = options[:advance_number].to_i + process_number + 1
+        n == 0 ? '' : n
+      end
+
+      def self.summarize_results(results)
+        sums = sum_up_results(results)
+        sums.sort.map{|word, number|  "#{number} #{word}#{'s' if number != 1}" }.join(', ')
+      end
+
+      protected
+
+      def self.sum_up_results(results)
+        results = results.join(' ').gsub(/s\b/,'') # combine and singularize results
+        counts = results.scan(/(\d+) (\w+)/)
+        sums = counts.inject(Hash.new(0)) do |sum, (number, word)|
+          sum[word] += number.to_i
+          sum
+        end
+
+        sums
+      end
+
+      # read output of the process and print it in chunks
+      def self.fetch_output(process)
+        all = ''
+        while buffer = process.readpartial(1000000)
+          all << buffer
+          $stdout.print buffer
+          $stdout.flush
+        end rescue EOFError
+
+        all
+      end
+
+      def self.with_runtime_info(tests)
+        lines = File.read(runtime_log).split("\n") rescue []
+
+        # use recorded test runtime if we got enough data
+        if lines.size * 1.5 > tests.size
+          puts "Using recorded test runtime"
+          times = Hash.new(1)
+          lines.each do |line|
+            test, time = line.split(":")
+            next unless test and time
+            times[File.expand_path(test)] = time.to_f
+          end
+          tests.sort.map{|test| [test, times[File.expand_path(test)]] }
+        else # use file sizes
+          tests.sort.map{|test| [test, File.stat(test).size] }
+        end
+      end
+
+      def self.find_tests(tests, options = {})
+        (tests || []).map do |file_or_folder|
+          if File.directory?(file_or_folder)
+            files = files_in_folder(file_or_folder, options)
+            files.grep(/#{Regexp.escape test_suffix}$/).grep(options[:pattern]||//)
+          else
+            file_or_folder
+          end
+        end.flatten.uniq
+      end
+
+      def self.files_in_folder(folder, options={})
+        pattern = if options[:symlinks] == false # not nil or true
+          "**/*"
+        else
+          # follow one symlink and direct children
+          # http://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
+          "**{,/*/**}/*"
+        end
+        Dir[File.join(folder, pattern)].uniq
+      end
+    end
+  end
+end

data/lib/parallel_tests/test/runtime_logger.rb

@@ -0,0 +1,92 @@
+require 'parallel_tests/test/runner'
+
+module ParallelTests
+  module Test
+    class RuntimeLogger
+      @@has_started = false
+
+      def self.log(test, start_time, end_time)
+        return if test.is_a? ::Test::Unit::TestSuite # don't log for suites-of-suites
+
+        if !@@has_started # make empty log file
+          File.open(logfile, 'w'){}
+          @@has_started = true
+        end
+
+        locked_appending_to(logfile) do |file|
+          file.puts(message(test, start_time, end_time))
+        end
+      end
+
+      def self.message(test, start_time, end_time)
+        delta = "%.2f" % (end_time.to_f-start_time.to_f)
+        filename = class_directory(test.class) + class_to_filename(test.class) + ".rb"
+        "#{filename}:#{delta}"
+      end
+
+      # Note: this is a best guess at conventional test directory structure, and may need
+      # tweaking / post-processing to match correctly for any given project
+      def self.class_directory(suspect)
+        result = "test/"
+
+        if defined?(Rails)
+          result += case suspect.superclass.name
+          when "ActionDispatch::IntegrationTest"
+            "integration/"
+          when "ActionDispatch::PerformanceTest"
+            "performance/"
+          when "ActionController::TestCase"
+            "functional/"
+          when "ActionView::TestCase"
+            "unit/helpers/"
+          else
+            "unit/"
+          end
+        end
+        result
+      end
+
+      # based on https://github.com/grosser/single_test/blob/master/lib/single_test.rb#L117
+      def self.class_to_filename(suspect)
+        word = suspect.to_s.dup
+        return word unless word.match /^[A-Z]/ and not word.match %r{/[a-z]}
+
+        word.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
+        word.gsub!(/\:\:/, '/')
+        word.tr!("-", "_")
+        word.downcase!
+        word
+      end
+
+      def self.locked_appending_to(file)
+        File.open(file, 'a') do |f|
+          begin
+            f.flock File::LOCK_EX
+            yield f
+          ensure
+            f.flock File::LOCK_UN
+          end
+        end
+      end
+
+      def self.logfile
+        ParallelTests::Test::Runner.runtime_log
+      end
+    end
+  end
+end
+
+require 'test/unit/testsuite'
+class ::Test::Unit::TestSuite
+  alias :run_without_timing :run unless defined? @@timing_installed
+
+  def run(result, &progress_block)
+    start_time=Time.now
+    run_without_timing(result, &progress_block)
+    end_time=Time.now
+    ParallelTests::Test::RuntimeLogger.log(self.tests.first, start_time, end_time)
+  end
+
+  @@timing_installed = true
+end

data/lib/parallel_tests/version.rb

@@ -0,0 +1,3 @@
+module ParallelTests
+  VERSION = Version = '0.9.0'
+end

data/parallel_tests.gemspec

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+name = "friendlyfashion-parallel_tests"
+require "parallel_tests/version"
+
+Gem::Specification.new name, ParallelTests::VERSION do |s|
+  s.summary = "Run Test::Unit / RSpec / Cucumber in parallel"
+  s.authors = ["Laurynas Butkus", "Tomas Varaneckas", "Justas Janauskas"]
+  s.email = ["laurynas.butkus@gmail.com", "tomas.varaneckas@gmail.com", "jjanauskas@gmail.com"]
+  s.homepage = "http://github.com/friendlyfashion/parallel_tests"
+  s.files = `git ls-files`.split("\n")
+  s.license = "MIT"
+  s.executables = ["parallel_cucumber", "parallel_rspec", "parallel_test"]
+  s.add_runtime_dependency "parallel"
+end

data/spec/integration_spec.rb

@@ -0,0 +1,244 @@
+require 'spec_helper'
+
+describe 'CLI' do
+  before do
+    `rm -rf #{folder}`
+  end
+
+  after do
+    `rm -rf #{folder}`
+  end
+
+  def folder
+    "/tmp/parallel_tests_tests"
+  end
+
+  def write(file, content)
+    path = "#{folder}/#{file}"
+    ensure_folder File.dirname(path)
+    File.open(path, 'w'){|f| f.write content }
+    path
+  end
+
+  def read(file)
+    File.read "#{folder}/#{file}"
+  end
+
+  def bin_folder
+    "#{File.expand_path(File.dirname(__FILE__))}/../bin"
+  end
+
+  def executable(options={})
+    "#{bin_folder}/parallel_#{options[:type] || 'test'}"
+  end
+
+  def ensure_folder(folder)
+    `mkdir -p #{folder}` unless File.exist?(folder)
+  end
+
+  def run_tests(test_folder, options={})
+    ensure_folder folder
+    processes = "-n #{options[:processes]||2}" unless options[:processes] == false
+    command = "cd #{folder} && #{options[:export]} #{executable(options)} #{test_folder} #{processes} #{options[:add]} 2>&1"
+    result = `#{command}`
+    raise "FAILED #{command}\n#{result}" if $?.success? == !!options[:fail]
+    result
+  end
+
+  it "runs tests in parallel" do
+    write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}'
+    write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}'
+    result = run_tests "spec", :type => 'rspec'
+
+    # test ran and gave their puts
+    result.should include('TEST1')
+    result.should include('TEST2')
+
+    # all results present
+    result.scan('1 example, 0 failure').size.should == 2 # 2 results
+    result.scan('2 examples, 0 failures').size.should == 1 # 1 summary
+    result.scan(/Finished in \d+\.\d+ seconds/).size.should == 2
+    result.scan(/Took \d+\.\d+ seconds/).size.should == 1 # parallel summary
+  end
+
+  it "does not run any tests if there are none" do
+    write 'spec/xxx_spec.rb', '1'
+    result = run_tests "spec", :type => 'rspec'
+    result.should include('No examples found')
+    result.should include('Took')
+  end
+
+  it "fails when tests fail" do
+    write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}'
+    write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){1.should == 2}}'
+    result = run_tests "spec", :fail => true, :type => 'rspec'
+
+    result.scan('1 example, 1 failure').size.should == 1
+    result.scan('1 example, 0 failure').size.should == 1
+    result.scan('2 examples, 1 failure').size.should == 1
+  end
+
+  context "with given commands" do
+    it "can exec given commands with ENV['TEST_ENV_NUM']" do
+      result = `#{executable} -e 'ruby -e "print ENV[:TEST_ENV_NUMBER.to_s].to_i"' -n 4`
+      result.gsub('"','').split('').sort.should == %w[0 2 3 4]
+    end
+
+    it "can exec given command non-parallel" do
+      result = `#{executable} -e 'ruby -e "sleep(rand(10)/100.0); puts ENV[:TEST_ENV_NUMBER.to_s].inspect"' -n 4 --non-parallel`
+      result.split("\n").should == %w["" "2" "3" "4"]
+    end
+
+    it "exists with success if all sub-processes returned success" do
+      system("#{executable} -e 'cat /dev/null' -n 4").should == true
+    end
+
+    it "exists with failure if any sub-processes returned failure" do
+      system("#{executable} -e 'test -e xxxx' -n 4").should == false
+    end
+  end
+
+  it "runs through parallel_rspec" do
+    version = `#{executable} -v`
+    `#{bin_folder}/parallel_rspec -v`.should == version
+  end
+
+  it "runs through parallel_cucumber" do
+    version = `#{executable} -v`
+    `#{bin_folder}/parallel_cucumber -v`.should == version
+  end
+
+  it "runs with --group-by found" do
+    # it only tests that it does not blow up, as it did before fixing...
+    write "spec/x1_spec.rb", "puts '111'"
+    run_tests "spec", :type => 'rspec', :add => '--group-by found'
+  end
+
+  it "runs faster with more processes" do
+    2.times{|i|
+      write "spec/xxx#{i}_spec.rb",  'describe("it"){it("should"){sleep 5}}; $stderr.puts ENV["TEST_ENV_NUMBER"]'
+    }
+    t = Time.now
+    run_tests("spec", :processes => 2, :type => 'rspec')
+    expected = 10
+    (Time.now - t).should <= expected
+  end
+
+  it "can run with given files" do
+    write "spec/x1_spec.rb", "puts '111'"
+    write "spec/x2_spec.rb", "puts '222'"
+    write "spec/x3_spec.rb", "puts '333'"
+    result = run_tests "spec/x1_spec.rb spec/x3_spec.rb", :type => 'rspec'
+    result.should include('111')
+    result.should include('333')
+    result.should_not include('222')
+  end
+
+  it "runs successfully without any files" do
+    results = run_tests "", :type => 'rspec'
+    results.should include("2 processes for 0 specs")
+    results.should include("Took")
+  end
+
+  it "can run with test-options" do
+    write "spec/x1_spec.rb", "111"
+    write "spec/x2_spec.rb", "111"
+    result = run_tests "spec",
+      :add => "--test-options ' --version'",
+      :processes => 2,
+      :type => 'rspec'
+    result.should =~ /\d+\.\d+\.\d+.*\d+\.\d+\.\d+/m # prints version twice
+  end
+
+  it "runs with PARALLEL_TEST_PROCESSORS processes" do
+    processes = 5
+    processes.times{|i|
+      write "spec/x#{i}_spec.rb", "puts %{ENV-\#{ENV['TEST_ENV_NUMBER']}-}"
+    }
+    result = run_tests "spec",
+      :export => "PARALLEL_TEST_PROCESSORS=#{processes}",
+      :processes => processes,
+      :type => 'rspec'
+    result.scan(/ENV-.?-/).should =~ ["ENV--", "ENV-2-", "ENV-3-", "ENV-4-", "ENV-5-"]
+  end
+
+  it "filters test by given pattern and relative paths" do
+    write "spec/x_spec.rb", "puts 'XXX'"
+    write "spec/y_spec.rb", "puts 'YYY'"
+    write "spec/z_spec.rb", "puts 'ZZZ'"
+    result = run_tests "spec", :add => "-p '^spec/(x|z)'", :type => "rspec"
+    result.should include('XXX')
+    result.should_not include('YYY')
+    result.should include('ZZZ')
+  end
+
+  context "Test::Unit" do
+    it "runs" do
+      write "test/x1_test.rb", "require 'test/unit'; class XTest < Test::Unit::TestCase; def test_xxx; end; end"
+      result = run_tests("test")
+      result.should include('1 test')
+    end
+
+    it "passes test options" do
+      write "test/x1_test.rb", "require 'test/unit'; class XTest < Test::Unit::TestCase; def test_xxx; end; end"
+      result = run_tests("test", :add => '--test-options "-v"')
+      result.should include('test_xxx') # verbose output of every test
+    end
+
+    it "runs successfully without any files" do
+      results = run_tests("")
+      results.should include("2 processes for 0 tests")
+      results.should include("Took")
+    end
+  end
+
+  context "Cucumber" do
+    before do
+      write "features/steps/a.rb", "
+        Given('I print TEST_ENV_NUMBER'){ puts \"YOUR TEST ENV IS \#{ENV['TEST_ENV_NUMBER']}!\" }
+        And('I sleep a bit'){ sleep 0.2 }
+      "
+    end
+
+    it "passes TEST_ENV_NUMBER when running with pattern (issue #86)" do
+      write "features/good1.feature", "Feature: xxx\n  Scenario: xxx\n    Given I print TEST_ENV_NUMBER"
+      write "features/good2.feature", "Feature: xxx\n  Scenario: xxx\n    Given I print TEST_ENV_NUMBER"
+      write "features/b.feature", "Feature: xxx\n  Scenario: xxx\n    Given I FAIL"
+      write "features/steps/a.rb", "Given('I print TEST_ENV_NUMBER'){ puts \"YOUR TEST ENV IS \#{ENV['TEST_ENV_NUMBER']}!\" }"
+
+      result = run_tests "features", :type => "cucumber", :add => '--pattern good'
+
+      result.should include('YOUR TEST ENV IS 2!')
+      result.should include('YOUR TEST ENV IS !')
+      result.should_not include('I FAIL')
+    end
+
+    it "writes a runtime log" do
+      log = "tmp/parallel_runtime_cucumber.log"
+      write(log, "x")
+      2.times{|i|
+        # needs sleep so that runtime loggers dont overwrite each other initially
+        write "features/good#{i}.feature", "Feature: xxx\n  Scenario: xxx\n    Given I print TEST_ENV_NUMBER\n    And I sleep a bit"
+      }
+      run_tests "features", :type => "cucumber"
+      read(log).gsub(/\.\d+/,'').split("\n").should =~ [
+        "features/good0.feature:0",
+        "features/good1.feature:0"
+      ]
+    end
+
+    it "runs each feature once when there are more processes then features (issue #89)" do
+      2.times{|i|
+        write "features/good#{i}.feature", "Feature: xxx\n  Scenario: xxx\n    Given I print TEST_ENV_NUMBER"
+      }
+      result = run_tests "features", :type => "cucumber", :add => '-n 3'
+      result.scan(/YOUR TEST ENV IS \d?!/).sort.should == ["YOUR TEST ENV IS !", "YOUR TEST ENV IS 2!"]
+    end
+
+    it "runs successfully without any files" do
+      results = run_tests("", :type => "cucumber")
+      results.should include("2 processes for 0 features")
+      results.should include("Took")
+    end
+  end
+end