parallel_tests 0.6.20 → 0.7.0.alpha

data/.gitignore

@@ -0,0 +1 @@
+*.sh

data/Gemfile

@@ -1,11 +1,9 @@
 source :rubygems
+gemspec
 
-gem 'parallel'
-
-group :dev do
+group :development do
   gem 'test-unit', :platform => :ruby_19
   gem 'rspec', '>=2.4'
   gem 'cucumber'
   gem 'rake'
-  gem 'jeweler'
 end

data/Gemfile.lock

@@ -1,3 +1,9 @@
+PATH
+  remote: .
+  specs:
+    parallel_tests (0.7.0.alpha)
+      parallel
+
 GEM
   remote: http://rubygems.org/
   specs:
@@ -11,11 +17,6 @@ GEM
     diff-lcs (1.1.2)
     gherkin (2.7.6)
       json (>= 1.4.6)
-    git (1.2.5)
-    jeweler (1.6.3)
-      bundler (~> 1.0)
-      git (>= 1.2.5)
-      rake
     json (1.6.4)
     parallel (0.5.1)
     rake (0.8.7)
@@ -35,8 +36,7 @@ PLATFORMS
 
 DEPENDENCIES
   cucumber
-  jeweler
-  parallel
+  parallel_tests!
   rake
   rspec (>= 2.4)
   test-unit

data/Rakefile

@@ -1,20 +1,22 @@
-task :default => :spec
-require "rspec/core/rake_task"
-RSpec::Core::RakeTask.new(:spec) do |t|
-  t.rspec_opts = '--backtrace --color'
+require 'bundler/gem_tasks'
+
+task :default do
+  sh "rspec spec/"
 end
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name     = "parallel_tests"
-    gem.summary  = "Run tests / specs / features in parallel"
-    gem.email    = "grosser.michael@gmail.com"
-    gem.homepage = "http://github.com/grosser/#{gem.name}"
-    gem.authors  = "Michael Grosser"
-  end
+# extracted from https://github.com/grosser/project_template
+rule /^version:bump:.*/ do |t|
+  sh "git status | grep 'nothing to commit'" # ensure we are not dirty
+  index = ['major', 'minor','patch'].index(t.name.split(':').last)
+  file = 'lib/GEM_NAME/version.rb'
+
+  version_file = File.read(file)
+  old_version, *version_parts = version_file.match(/(\d+)\.(\d+)\.(\d+)/).to_a
+  version_parts[index] = version_parts[index].to_i + 1
+  version_parts[2] = 0 if index < 2 # remove patch for minor
+  version_parts[1] = 0 if index < 1 # remove minor for major
+  new_version = version_parts * '.'
+  File.open(file,'w'){|f| f.write(version_file.sub(old_version, new_version)) }
 
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install jeweler"
+  sh "bundle && git add #{file} Gemfile.lock && git commit -m 'bump version to #{new_version}'"
 end

data/Readme.md

@@ -89,16 +89,16 @@ Even process runtimes
 
 Log test runtime to give each process the same runtime.
 
-Rspec: Add to your `spec/parallel_spec.opts` (or `spec/spec.opts`) :
+Rspec: Add to your `.rspec_parallel` (or `.rspec`) :
 
     RSpec 1.x:
       --format progress
-      --require parallel_specs/spec_runtime_logger
-      --format ParallelSpecs::SpecRuntimeLogger:tmp/parallel_profile.log
+      --require parallel_tests/spec/runtime_logger
+      --format ParallelTests::Spec::RuntimeLogger:tmp/parallel_profile.log
     RSpec >= 2.4:
       If installed as plugin: -I vendor/plugins/parallel_tests/lib
       --format progress
-      --format ParallelSpecs::SpecRuntimeLogger --out tmp/parallel_profile.log
+      --format ParallelTests::Spec::RuntimeLogger --out tmp/parallel_profile.log
 
 Test::Unit:  Add to your `test_helper.rb`:
     require 'parallel_tests/runtime_logger'
@@ -109,16 +109,16 @@ SpecSummaryLogger
 
 This logger logs the test output without the different processes overwriting each other.
 
-Add the following to your `spec/parallel_spec.opts` (or `spec/spec.opts`) :
+Add the following to your `.rspec_parallel` (or `.rspec`) :
 
     RSpec 1.x:
       --format progress
-      --require parallel_specs/spec_summary_logger
-      --format ParallelSpecs::SpecSummaryLogger:tmp/spec_summary.log
+      --require parallel_tests/spec/summary_logger
+      --format ParallelTests::Spec::SummaryLogger:tmp/spec_summary.log
     RSpec >= 2.2:
       If installed as plugin: -I vendor/plugins/parallel_tests/lib
       --format progress
-      --format ParallelSpecs::SpecSummaryLogger --out tmp/spec_summary.log
+      --format ParallelTests::Spec::SummaryLogger --out tmp/spec_summary.log
 
 SpecFailuresLogger
 -----------------------
@@ -129,16 +129,16 @@ E.g.
 
     rspec /path/to/my_spec.rb:123 # should do something
 
-Add the following to your `spec/parallel_spec.opts` (or `spec/spec.opts`) :
+Add the following to your `.rspec_parallel` (or `.rspec`) :
 
     RSpec 1.x:
       --format progress
-      --require parallel_specs/spec_failures_logger
-      --format ParallelSpecs::SpecFailuresLogger:tmp/failing_specs.log
+      --require parallel_tests/spec/failures_logger
+      --format ParallelTests::Spec::FailuresLogger:tmp/failing_specs.log
     RSpec >= 2.4:
       If installed as plugin: -I vendor/plugins/parallel_tests/lib
       --format progress
-      --format ParallelSpecs::SpecFailuresLogger --out tmp/failing_specs.log
+      --format ParallelTests::Spec::FailuresLogger --out tmp/failing_specs.log
 
 Setup for non-rails
 ===================
@@ -200,6 +200,9 @@ TIPS
 
 TODO
 ====
+ - move everything Rails 2 related to a e.g. Rails2Readme.md and link it
+ - unify runtime-log location
+ - add tests for cucumber runtime formatter
  - make jRuby compatible [basics](http://yehudakatz.com/2009/07/01/new-rails-isolation-testing/)
  - make windows compatible
 

data/bin/parallel_test

@@ -1,101 +1,5 @@
 #!/usr/bin/env ruby
-require 'rubygems'
-require 'optparse'
-require 'parallel'
-raise "please ' gem install parallel '" if Gem::Version.new(Parallel::VERSION) < Gem::Version.new('0.4.2')
 $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
 require "parallel_tests"
-
-options = {}
-OptionParser.new do |opts|
-  opts.banner = <<BANNER
-Run all tests in parallel, giving each process ENV['TEST_ENV_NUMBER'] ('', '2', '3', ...)
-
-[optional] Only run selected files & folders:
-    parallel_test test/bar test/baz/xxx_text.rb
-
-Options are:
-BANNER
-  opts.on("-n [PROCESSES]", Integer, "How many processes to use, default: available CPUs"){|n| options[:count] = n }
-  opts.on("-p", '--pattern [PATTERN]', "run tests matching this pattern"){|pattern| options[:pattern] = pattern }
-  opts.on("--no-sort", "do not sort files before running them"){ |no_sort| options[:no_sort] = no_sort }
-  opts.on("-m [FLOAT]", "--multiply-processes [FLOAT]", Float, "use given number as a multiplier of processes to run"){ |multiply| options[:multiply] = multiply }
-  opts.on("-r", '--root [PATH]', "execute test commands from this path"){|path| options[:root] = path }
-  opts.on("-s [PATTERN]", "--single [PATTERN]", "Run all matching files in only one process") do |pattern|
-    options[:single_process] ||= []
-    options[:single_process] << /#{pattern}/
-  end
-  opts.on("-e", '--exec [COMMAND]', "execute this code parallel and with ENV['TEST_ENV_NUM']"){|path| options[:execute] = path }
-  opts.on("-o", "--test-options '[OPTIONS]'", "execute test commands with those options"){|arg| options[:test_options] = arg }
-  opts.on("-t", "--type [TYPE]", "which type of tests to run? test, spec or features"){|type| options[:type] = type }
-  opts.on("--non-parallel", "execute same commands but do not in parallel, needs --exec"){ options[:non_parallel] = true }
-  opts.on("--chunk-timeout [TIMEOUT]", "timeout before re-printing the output of a child-process"){|timeout| options[:chunk_timeout] = timeout.to_f }
-  opts.on('-v', '--version', 'Show Version'){ puts ParallelTests::VERSION; exit}
-  opts.on("-h", "--help", "Show this.") { puts opts; exit }
-end.parse!
-
-raise "--no-sort and --single-process are not supported" if options[:no_sort] and options[:single_process]
-
-# get files to run from arguments
-options[:files] = ARGV if ARGV.size > 0
-
-num_processes = ParallelTests.determine_number_of_processes(options[:count])
-num_processes = num_processes * (options[:multiply] || 1)
-
-if options[:execute]
-  runs = (0...num_processes).to_a
-  results = if options[:non_parallel]
-    runs.map do |i|
-      ParallelTests.execute_command(options[:execute], i, options)
-    end
-  else
-    Parallel.map(runs, :in_processes => num_processes) do |i|
-      ParallelTests.execute_command(options[:execute], i, options)
-    end
-  end.flatten
-  abort if results.any?{|r| r[:exit_status] != 0 }
-else
-  lib, name, task = {
-    'test' => ["tests", "test", "test"],
-    'spec' => ["specs", "spec", "spec"],
-    'features' => ["cucumber", "feature", "features"]
-  }[options[:type]||'test']
-
-  require "parallel_#{lib}"
-  klass = eval("Parallel#{lib.capitalize}")
-
-  start = Time.now
-
-  tests_folder = task
-  tests_folder = File.join(options[:root], tests_folder) unless options[:root].to_s.empty?
-
-  groups = klass.tests_in_groups(options[:files] || tests_folder, num_processes, options)
-  num_processes = groups.size
-
-  #adjust processes to groups
-  abort "no #{name}s found!" if groups.size == 0
-
-  num_tests = groups.inject(0){|sum,item| sum + item.size }
-  puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{num_tests / groups.size} #{name}s per process"
-
-  test_results = Parallel.map(groups, :in_processes => num_processes) do |group|
-    if group.empty?
-      {:stdout => '', :exit_status => 0}
-    else
-      klass.run_tests(group, groups.index(group), options)
-    end
-  end
-
-  #parse and print results
-  results = klass.find_results(test_results.map{|result| result[:stdout] }*"")
-  puts ""
-  puts klass.summarize_results(results)
-
-  #report total time taken
-  puts ""
-  puts "Took #{Time.now - start} seconds"
-
-  #exit with correct status code so rake parallel:test && echo 123 works
-  failed = test_results.any?{|result| result[:exit_status] != 0 }
-  abort "#{name.capitalize}s Failed" if failed
-end
+require "parallel_tests/cli"
+ParallelTest::CLI.run(ARGV)

data/lib/parallel_tests.rb

@@ -1,10 +1,8 @@
 require 'parallel'
+require 'parallel_tests/version'
 require 'parallel_tests/grouper'
-require 'parallel_tests/railtie'
-
-class ParallelTests
-  VERSION = File.read( File.join(File.dirname(__FILE__),'..','VERSION') ).strip
 
+module ParallelTests
   def self.determine_number_of_processes(count)
     [
       count,
@@ -29,88 +27,6 @@ class ParallelTests
     [num_processes, pattern.to_s, options.to_s]
   end
 
-  # finds all tests and partitions them into groups
-  def self.tests_in_groups(root, num_groups, options={})
-    tests = find_tests(root, options)
-    if options[:no_sort] == true
-      Grouper.in_groups(tests, num_groups)
-    else
-      tests = with_runtime_info(tests)
-      Grouper.in_even_groups_by_size(tests, num_groups, options)
-    end
-  end
-
-  def self.run_tests(test_files, process_number, options)
-    require_list = test_files.map { |filename| %{"#{File.expand_path filename}"} }.join(",")
-    cmd = "ruby -Itest -e '[#{require_list}].each {|f| require f }' -- #{options[:test_options]}"
-    execute_command(cmd, process_number, options)
-  end
-
-  def self.execute_command(cmd, process_number, options)
-    cmd = "TEST_ENV_NUMBER=#{test_env_number(process_number)} ; export TEST_ENV_NUMBER; #{cmd}"
-    f = open("|#{cmd}", 'r')
-    output = fetch_output(f, options)
-    f.close
-    {:stdout => output, :exit_status => $?.exitstatus}
-  end
-
-  def self.find_results(test_output)
-    test_output.split("\n").map {|line|
-      line = line.gsub(/\.|F|\*/,'')
-      next unless line_is_result?(line)
-      line
-    }.compact
-  end
-
-  def self.test_env_number(process_number)
-    process_number == 0 ? '' : process_number + 1
-  end
-
-  def self.runtime_log
-    'tmp/parallel_runtime_test.log'
-  end
-
-  def self.summarize_results(results)
-    results = results.join(' ').gsub(/s\b/,'') # combine and singularize results
-    counts = results.scan(/(\d+) (\w+)/)
-    sums = counts.inject(Hash.new(0)) do |sum, (number, word)|
-      sum[word] += number.to_i
-      sum
-    end
-    sums.sort.map{|word, number|  "#{number} #{word}#{'s' if number != 1}" }.join(', ')
-  end
-
-  protected
-
-  # read output of the process and print in in chucks
-  def self.fetch_output(process, options)
-    all = ''
-    buffer = ''
-    timeout = options[:chunk_timeout] || 0.2
-    flushed = Time.now.to_f
-
-    while char = process.getc
-      char = (char.is_a?(Fixnum) ? char.chr : char) # 1.8 <-> 1.9
-      all << char
-
-      # print in chunks so large blocks stay together
-      now = Time.now.to_f
-      buffer << char
-      if flushed + timeout < now
-        $stdout.print buffer
-        $stdout.flush
-        buffer = ''
-        flushed = now
-      end
-    end
-
-    # print the remainder
-    $stdout.print buffer
-    $stdout.flush
-
-    all
-  end
-
   # copied from http://github.com/carlhuda/bundler Bundler::SharedHelpers#find_gemfile
   def self.bundler_enabled?
     return true if Object.const_defined?(:Bundler)
@@ -126,43 +42,4 @@ class ParallelTests
 
     false
   end
-
-  def self.line_is_result?(line)
-    line =~ /\d+ failure/
-  end
-
-  def self.test_suffix
-    "_test.rb"
-  end
-
-  def self.with_runtime_info(tests)
-    lines = File.read(runtime_log).split("\n") rescue []
-
-    # use recorded test runtime if we got enough data
-    if lines.size * 1.5 > tests.size
-      puts "Using recorded test runtime"
-      times = Hash.new(1)
-      lines.each do |line|
-        test, time = line.split(":")
-        next unless test and time
-        times[File.expand_path(test)] = time.to_f
-      end
-      tests.sort.map{|test| [test, times[test]] }
-    else # use file sizes
-      tests.sort.map{|test| [test, File.stat(test).size] }
-    end
-  end
-
-  def self.find_tests(root, options={})
-    if root.is_a?(Array)
-      root
-    else
-      # follow one symlink and direct children
-      # http://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
-      files = Dir["#{root}/**{,/*/**}/*#{test_suffix}"].uniq
-      files = files.map{|f| f.sub(root+'/','') }
-      files = files.grep(/#{options[:pattern]}/)
-      files.map{|f| "#{root}/#{f}" }
-    end
-  end
 end

data/lib/parallel_tests/cli.rb

@@ -0,0 +1,102 @@
+require 'optparse'
+require 'parallel_tests/test/runner'
+
+module ParallelTest
+  module CLI
+    def self.run(argv)
+      options = {}
+      OptionParser.new do |opts|
+        opts.banner = <<BANNER
+Run all tests in parallel, giving each process ENV['TEST_ENV_NUMBER'] ('', '2', '3', ...)
+
+[optional] Only run selected files & folders:
+    parallel_test test/bar test/baz/xxx_text.rb
+
+Options are:
+BANNER
+        opts.on("-n [PROCESSES]", Integer, "How many processes to use, default: available CPUs") { |n| options[:count] = n }
+        opts.on("-p", '--pattern [PATTERN]', "run tests matching this pattern") { |pattern| options[:pattern] = pattern }
+        opts.on("--no-sort", "do not sort files before running them") { |no_sort| options[:no_sort] = no_sort }
+        opts.on("-m [FLOAT]", "--multiply-processes [FLOAT]", Float, "use given number as a multiplier of processes to run") { |multiply| options[:multiply] = multiply }
+        opts.on("-r", '--root [PATH]', "execute test commands from this path") { |path| options[:root] = path }
+        opts.on("-s [PATTERN]", "--single [PATTERN]", "Run all matching files in only one process") do |pattern|
+          options[:single_process] ||= []
+          options[:single_process] << /#{pattern}/
+        end
+        opts.on("-e", '--exec [COMMAND]', "execute this code parallel and with ENV['TEST_ENV_NUM']") { |path| options[:execute] = path }
+        opts.on("-o", "--test-options '[OPTIONS]'", "execute test commands with those options") { |arg| options[:test_options] = arg }
+        opts.on("-t", "--type [TYPE]", "which type of tests to run? test, spec or features") { |type| options[:type] = type }
+        opts.on("--non-parallel", "execute same commands but do not in parallel, needs --exec") { options[:non_parallel] = true }
+        opts.on("--chunk-timeout [TIMEOUT]", "timeout before re-printing the output of a child-process") { |timeout| options[:chunk_timeout] = timeout.to_f }
+        opts.on('-v', '--version', 'Show Version') { puts ParallelTests::VERSION; exit }
+        opts.on("-h", "--help", "Show this.") { puts opts; exit }
+      end.parse!(argv)
+
+      raise "--no-sort and --single-process are not supported" if options[:no_sort] and options[:single_process]
+
+      # get files to run from arguments
+      options[:files] = argv if argv.size > 0
+
+      num_processes = ParallelTests.determine_number_of_processes(options[:count])
+      num_processes = num_processes * (options[:multiply] || 1)
+
+      if options[:execute]
+        runs = (0...num_processes).to_a
+        results = if options[:non_parallel]
+          runs.map do |i|
+            ParallelTests::Test::Runner.execute_command(options[:execute], i, options)
+          end
+        else
+          Parallel.map(runs, :in_processes => num_processes) do |i|
+            ParallelTests::Test::Runner.execute_command(options[:execute], i, options)
+          end
+        end.flatten
+        abort if results.any? { |r| r[:exit_status] != 0 }
+      else
+        lib, name, task = {
+          'test' => ["test", "test", "test"],
+          'spec' => ["spec", "spec", "spec"],
+          'features' => ["cucumber", "feature", "features"]
+        }[options[:type]||'test']
+
+        require "parallel_tests/#{lib}/runner"
+        klass = eval("ParallelTests::#{lib.capitalize}::Runner")
+
+        start = Time.now
+
+        tests_folder = task
+        tests_folder = File.join(options[:root], tests_folder) unless options[:root].to_s.empty?
+
+        groups = klass.tests_in_groups(options[:files] || tests_folder, num_processes, options)
+        num_processes = groups.size
+
+        #adjust processes to groups
+        abort "no #{name}s found!" if groups.size == 0
+
+        num_tests = groups.inject(0) { |sum, item| sum + item.size }
+        puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{num_tests / groups.size} #{name}s per process"
+
+        test_results = Parallel.map(groups, :in_processes => num_processes) do |group|
+          if group.empty?
+            {:stdout => '', :exit_status => 0}
+          else
+            klass.run_tests(group, groups.index(group), options)
+          end
+        end
+
+        #parse and print results
+        results = klass.find_results(test_results.map { |result| result[:stdout] }*"")
+        puts ""
+        puts klass.summarize_results(results)
+
+        #report total time taken
+        puts ""
+        puts "Took #{Time.now - start} seconds"
+
+        #exit with correct status code so rake parallel:test && echo 123 works
+        failed = test_results.any? { |result| result[:exit_status] != 0 }
+        abort "#{name.capitalize}s Failed" if failed
+      end
+    end
+  end
+end