parallelized_specs 0.0.1

data/lib/parallel_tests.rb

@@ -0,0 +1,163 @@
+require 'parallel'
+require 'parallel_tests/grouper'
+require 'parallel_tests/railtie'
+
+class ParallelTests
+  VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
+
+  # parallel:spec[:count, :pattern, :options]
+  def self.parse_rake_args(args)
+    # order as given by user
+    args = [args[:count], args[:pattern], args[:options]]
+
+    # count given or empty ?
+    # parallel:spec[2,models,options]
+    # parallel:spec[,models,options]
+    count = args.shift if args.first.to_s =~ /^\d*$/
+    num_processes = count.to_i unless count.to_s.empty?
+    num_processes ||= ENV['PARALLEL_TEST_PROCESSORS'].to_i if ENV['PARALLEL_TEST_PROCESSORS']
+    num_processes ||= Parallel.processor_count
+
+    pattern = args.shift
+    options = args.shift
+
+    [num_processes.to_i, pattern.to_s, options.to_s]
+  end
+
+  # finds all tests and partitions them into groups
+  def self.tests_in_groups(root, num_groups, options={})
+    tests = find_tests(root, options)
+    if options[:no_sort] == true
+      Grouper.in_groups(tests, num_groups)
+    else
+      tests = with_runtime_info(tests)
+      Grouper.in_even_groups_by_size(tests, num_groups, options)
+    end
+  end
+
+  def self.run_tests(test_files, process_number, options)
+    require_list = test_files.map { |filename| %{"#{File.expand_path filename}"} }.join(",")
+    cmd = "ruby -Itest -e '[#{require_list}].each {|f| require f }' -- #{options[:test_options]}"
+    execute_command(cmd, process_number, options)
+  end
+
+  def self.execute_command(cmd, process_number, options)
+    cmd = "TEST_ENV_NUMBER=#{test_env_number(process_number)} ; export TEST_ENV_NUMBER; #{cmd}"
+    f = open("|#{cmd}", 'r')
+    output = fetch_output(f, options)
+    f.close
+    {:stdout => output, :exit_status => $?.exitstatus}
+  end
+
+  def self.find_results(test_output)
+    test_output.split("\n").map {|line|
+      line = line.gsub(/\.|F|\*/,'')
+      next unless line_is_result?(line)
+      line
+    }.compact
+  end
+
+  def self.test_env_number(process_number)
+    process_number == 0 ? '' : process_number + 1
+  end
+
+  def self.runtime_log
+    'tmp/parallel_runtime_test.log'
+  end
+
+  def self.summarize_results(results)
+    results = results.join(' ').gsub(/s\b/,'') # combine and singularize results
+    counts = results.scan(/(\d+) (\w+)/)
+    sums = counts.inject(Hash.new(0)) do |sum, (number, word)|
+      sum[word] += number.to_i
+      sum
+    end
+    sums.sort.map{|word, number|  "#{number} #{word}#{'s' if number != 1}" }.join(', ')
+  end
+
+  protected
+
+  # read output of the process and print in in chucks
+  def self.fetch_output(process, options)
+    all = ''
+    buffer = ''
+    timeout = options[:chunk_timeout] || 0.2
+    flushed = Time.now.to_f
+
+    while char = process.getc
+      char = (char.is_a?(Fixnum) ? char.chr : char) # 1.8 <-> 1.9
+      all << char
+
+      # print in chunks so large blocks stay together
+      now = Time.now.to_f
+      buffer << char
+      if flushed + timeout < now
+        print buffer
+        STDOUT.flush
+        buffer = ''
+        flushed = now
+      end
+    end
+
+    # print the remainder
+    print buffer
+    STDOUT.flush
+
+    all
+  end
+
+  # copied from http://github.com/carlhuda/bundler Bundler::SharedHelpers#find_gemfile
+  def self.bundler_enabled?
+    return true if Object.const_defined?(:Bundler)
+
+    previous = nil
+    current = File.expand_path(Dir.pwd)
+
+    until !File.directory?(current) || current == previous
+      filename = File.join(current, "Gemfile")
+      return true if File.exists?(filename)
+      current, previous = File.expand_path("..", current), current
+    end
+
+    false
+  end
+
+  def self.line_is_result?(line)
+    line =~ /\d+ failure/
+  end
+
+  def self.test_suffix
+    "_test.rb"
+  end
+
+  def self.with_runtime_info(tests)
+    lines = File.read(runtime_log).split("\n") rescue []
+
+    # use recorded test runtime if we got enough data
+    if lines.size * 1.5 > tests.size
+      puts "Using recorded test runtime"
+      times = Hash.new(1)
+      lines.each do |line|
+        test, time = line.split(":")
+        next unless test and time
+        times[File.expand_path(test)] = time.to_f
+      end
+      tests.sort.map{|test| [test, times[test]] }
+    else # use file sizes
+      tests.sort.map{|test| [test, File.stat(test).size] }
+    end
+  end
+
+  def self.find_tests(root, options={})
+    if root.is_a?(Array)
+      root
+    else
+      # follow one symlink and direct children
+      # http://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
+      files = Dir["#{root}/**{,/*/**}/*#{test_suffix}"].uniq
+      files = files.map{|f| f.sub(root+'/','') }
+      files = files.grep(/#{options[:pattern]}/)
+      files.map{|f| "#{root}/#{f}" }
+    end
+  end
+end

data/lib/tasks/parallel_tests.rake

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), "/../parallel_tests/tasks")

data/parallelized_specs.gemspec

@@ -0,0 +1,64 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "parallelized_specs"
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jake Sorce, Bryan Madsen"]
+  s.date = "2012-04-17"
+  s.email = "jake@instructure.com"
+  s.executables = ["parallel_test", "parallel_spec"]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "Rakefile",
+    "Readme.md",
+    "VERSION",
+    "bin/parallel_spec",
+    "bin/parallel_test",
+    "lib/parallel_specs.rb",
+    "lib/parallel_specs/spec_error_count_logger.rb",
+    "lib/parallel_specs/spec_error_logger.rb",
+    "lib/parallel_specs/spec_failures_logger.rb",
+    "lib/parallel_specs/spec_logger_base.rb",
+    "lib/parallel_specs/spec_runtime_logger.rb",
+    "lib/parallel_specs/spec_start_finish_logger.rb",
+    "lib/parallel_specs/spec_summary_logger.rb",
+    "lib/parallel_tests.rb",
+    "lib/parallel_tests/grouper.rb",
+    "lib/parallel_tests/railtie.rb",
+    "lib/parallel_tests/runtime_logger.rb",
+    "lib/parallel_tests/tasks.rb",
+    "lib/tasks/parallel_tests.rake",
+    "parallelized_specs.gemspec",
+    "spec/integration_spec.rb",
+    "spec/parallel_specs/spec_failure_logger_spec.rb",
+    "spec/parallel_specs/spec_runtime_logger_spec.rb",
+    "spec/parallel_specs/spec_summary_logger_spec.rb",
+    "spec/parallel_specs_spec.rb",
+    "spec/parallel_tests/runtime_logger_spec.rb",
+    "spec/parallel_tests_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://github.com/jake/parallelized_specs"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.22"
+  s.summary = "Run rspec tests in parallel"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<parallel>, [">= 0"])
+    else
+      s.add_dependency(%q<parallel>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<parallel>, [">= 0"])
+  end
+end
+

data/spec/integration_spec.rb

@@ -0,0 +1,133 @@
+require 'spec_helper'
+
+describe 'CLI' do
+  before do
+    `rm -rf #{folder}`
+  end
+
+  after do
+    `rm -rf #{folder}`
+  end
+
+  def folder
+    "/tmp/parallel_tests_tests"
+  end
+
+  def write(file, content)
+    path = "#{folder}/#{file}"
+    `mkdir -p #{File.dirname(path)}` unless File.exist?(File.dirname(path))
+    File.open(path, 'w'){|f| f.write content }
+    path
+  end
+
+  def bin_folder
+    "#{File.expand_path(File.dirname(__FILE__))}/../bin"
+  end
+
+  def executable
+    "#{bin_folder}/parallel_test"
+  end
+
+  def run_tests(options={})
+    `cd #{folder} && #{executable} --chunk-timeout 999 -t #{options[:type] || 'spec'} -n #{options[:processes]||2} #{options[:add]} 2>&1`
+  end
+
+  it "runs tests in parallel" do
+    write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}'
+    write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}'
+    result = run_tests
+
+    # test ran and gave their puts
+    result.should include('TEST1')
+    result.should include('TEST2')
+
+    # all results present
+    result.scan('1 example, 0 failure').size.should == 2 # 2 results
+    result.scan('2 examples, 0 failures').size.should == 1 # 1 summary
+    result.scan(/Finished in \d+\.\d+ seconds/).size.should == 2
+    result.scan(/Took \d+\.\d+ seconds/).size.should == 1 # parallel summary
+    $?.success?.should == true
+  end
+
+  it "does not run any tests if there are none" do
+    write 'spec/xxx.rb', 'xxx'
+    result = run_tests
+    result.should include('No examples found')
+    result.should include('Took')
+  end
+
+  it "fails when tests fail" do
+    write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}'
+    write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){1.should == 2}}'
+    result = run_tests
+
+    result.scan('1 example, 1 failure').size.should == 1
+    result.scan('1 example, 0 failure').size.should == 1
+    result.scan('2 examples, 1 failure').size.should == 1
+    $?.success?.should == false
+  end
+
+  it "can exec given commands with ENV['TEST_ENV_NUM']" do
+    result = `#{executable} -e 'ruby -e "print ENV[:TEST_ENV_NUMBER.to_s].to_i"' -n 4`
+    result.gsub('"','').split('').sort.should == %w[0 2 3 4]
+  end
+
+  it "can exec given command non-parallel" do
+    result = `#{executable} -e 'ruby -e "sleep(rand(10)/100.0); puts ENV[:TEST_ENV_NUMBER.to_s].inspect"' -n 4 --non-parallel`
+    result.split("\n").should == %w["" "2" "3" "4"]
+  end
+
+  it "exists with success if all sub-processes returned success" do
+    system("#{executable} -e 'cat /dev/null' -n 4").should == true
+  end
+
+  it "exists with failure if any sub-processes returned failure" do
+    system("#{executable} -e 'test -e xxxx' -n 4").should == false
+  end
+
+  it "can run through parallel_spec / parallel_cucumber" do
+    version = `#{executable} -v`
+    `#{bin_folder}/parallel_spec -v`.should == version
+    `#{bin_folder}/parallel_cucumber -v`.should == version
+  end
+
+  it "runs faster with more processes" do
+    2.times{|i|
+      write "spec/xxx#{i}_spec.rb",  'describe("it"){it("should"){sleep 5}}; $stderr.puts ENV["TEST_ENV_NUMBER"]'
+    }
+    t = Time.now
+    run_tests(:processes => 2)
+    expected = 10
+    (Time.now - t).should <= expected
+  end
+
+  it "can can with given files" do
+    write "spec/x1_spec.rb", "puts '111'"
+    write "spec/x2_spec.rb", "puts '222'"
+    write "spec/x3_spec.rb", "puts '333'"
+    result = run_tests(:add => 'spec/x1_spec.rb spec/x3_spec.rb')
+    result.should include('111')
+    result.should include('333')
+    result.should_not include('222')
+  end
+
+  it "can run with test-options" do
+    write "spec/x1_spec.rb", ""
+    write "spec/x2_spec.rb", ""
+    result = run_tests(:add => "--test-options ' --version'", :processes => 2)
+    result.should =~ /\d+\.\d+\.\d+.*\d+\.\d+\.\d+/m # prints version twice
+  end
+
+  it "runs with test::unit" do
+    write "test/x1_test.rb", "require 'test/unit'; class XTest < Test::Unit::TestCase; def test_xxx; end; end"
+    result = run_tests(:type => :test)
+    result.should include('1 test')
+    $?.success?.should == true
+  end
+
+  it "passes test options to test::unit" do
+    write "test/x1_test.rb", "require 'test/unit'; class XTest < Test::Unit::TestCase; def test_xxx; end; end"
+    result = run_tests(:type => :test, :add => '--test-options "-v"')
+    result.should include('test_xxx') # verbose output of every test
+  end
+end

data/spec/parallel_specs/spec_failure_logger_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+describe ParallelSpecs::SpecFailuresLogger do
+  def silence_warnings
+    old_verbose, $VERBOSE = $VERBOSE, nil
+    yield
+  ensure
+    $VERBOSE = old_verbose
+  end
+
+  before do
+    @output     = OutputLogger.new([])
+    @example1   = mock( 'example', :location => "#{Dir.pwd}/spec/path/to/example:123", :full_description => 'should do stuff', :description => 'd' )
+    @example2   = mock( 'example', :location => "#{Dir.pwd}/spec/path/to/example2:456", :full_description => 'should do other stuff', :description => 'd')
+    @exception1 = mock( :to_s => 'exception', :backtrace => [ '/path/to/error/line:33' ] )
+    @failure1   = mock( 'example', :location => "#{Dir.pwd}/example:123", :header => 'header', :exception => @exception1 )
+    @logger = ParallelSpecs::SpecFailuresLogger.new( @output )
+  end
+
+  after do
+    silence_warnings{ ParallelSpecs::SpecLoggerBase::RSPEC_1 = false }
+  end
+
+  def clean_output
+    @output.output.join("\n").gsub(/\e\[\d+m/,'')
+  end
+
+  it "should produce a list of command lines for failing examples" do
+    @logger.example_failed @example1
+    @logger.example_failed @example2
+
+    @logger.dump_failures
+    @logger.dump_summary(1,2,3,4)
+
+    clean_output.should =~ /^rspec .*? should do stuff/
+    clean_output.should =~ /^rspec .*? should do other stuff/
+  end
+
+  it "should invoke spec for rspec 1" do
+    silence_warnings{ ParallelSpecs::SpecLoggerBase::RSPEC_1 = true }
+    ParallelSpecs.stub!(:bundler_enabled?).and_return true
+    ParallelSpecs.stub!(:run).with("bundle show rspec").and_return "/foo/bar/rspec-1.0.2"
+    @logger.example_failed @example1
+
+    @logger.dump_failures
+    @logger.dump_summary(1,2,3,4)
+
+    clean_output.should =~ /^bundle exec spec/
+  end
+
+  it "should invoke rspec for rspec 2" do
+    ParallelSpecs.stub!(:bundler_enabled?).and_return true
+    ParallelSpecs.stub!(:run).with("bundle show rspec").and_return "/foo/bar/rspec-2.0.2"
+    @logger.example_failed @example1
+
+    @logger.dump_failures
+    @logger.dump_summary(1,2,3,4)
+
+    clean_output.should =~ /^rspec/
+  end
+
+  it "should return relative paths" do
+    @logger.example_failed @example1
+    @logger.example_failed @example2
+
+    @logger.dump_failures
+    @logger.dump_summary(1,2,3,4)
+
+    clean_output.should =~ %r(\./spec/path/to/example:123)
+    clean_output.should =~ %r(\./spec/path/to/example2:456)
+  end
+
+
+  # should not longer be a problem since its using native rspec methods
+  xit "should not log examples without location" do
+    example = mock('example', :location => 'bla', :full_description => 'before :all')
+    @logger.example_failed example
+    @logger.dump_failures
+    @logger.dump_summary(1,2,3,4)
+    clean_output.should == ''
+  end
+end