snap_ci-parallel_tests 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64a3cfc18f6ec422dcec8eb046042c4cea62e645
-  data.tar.gz: e93d0f1bbd0c1318f185867dcf89c3f431983611
+  metadata.gz: aa9a82a7f30c6776d93df92ed4fe8e3672a31a19
+  data.tar.gz: 9bb5b4e69777aaa30185b0527ed769945b3b28b1
 SHA512:
-  metadata.gz: d735091f3c891b157948e2642edbc69b1c57e23e36543299c2696d2a0ed4a85739d4e8dd084614a88c2bcb93a07c23fe40c6001249dac15e704d3c8971dccfbb
-  data.tar.gz: 4920b30f0b069989337624808180f995ce4d320f2ef5bbe4cc602d18dc107c97098f8d2fa22973eeb4726eb669222ffb974e0d82556bfc6874023ced4ffa84bc
+  metadata.gz: 4133669bf0b3a6696a15d24225b739960904a27b32b7e0516754f8703022910920027bdc35ad8210efb7ce346647f74ef30c3540fa0da94738d6aad3bc9671ca
+  data.tar.gz: 020b61c12f3632dfa24e22e76f37388daafa729d2c70c9c67661d26eca750fa3d3151f34881971fcd8979b971fae734e924e2cf0409efdf592bcdf86067ab1cf

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    snap_ci-parallel_tests (0.0.1)
+    snap_ci-parallel_tests (0.0.2)
 
 GEM
   remote: https://rubygems.org/
@@ -68,7 +68,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.7)
+  bundler (~> 1.4)
   rake
   rspec (>= 2.12.0)
   rspec-rails (>= 2.12.0)

data/README.md

@@ -1,4 +1,4 @@
-# SnapCI::ParallelTests
+# SnapCI::ParallelTests [![Build Status](https://snap-ci.com/snap-ci/parallel-tests/branch/master/build_image)](https://snap-ci.com/snap-ci/parallel-tests/branch/master)
 
 Run Test::Unit / RSpec in parallel across multiple workers on [Snap CI](https://snap-ci.com).
 
@@ -22,23 +22,23 @@ Or install it yourself as:
 
 Depending on the framework of your choice -
 
-    $ bundle exec snap_ci-parallel_rspec [options] [files or directories] [-- [rspec options]]
-    $ bundle exec snap_ci-parallel_test [options] [files or directories] [-- [Test::Unit or MiniTest options]]
+    $ bundle exec snap-ci-parallel-rspec [options] [files or directories] [-- [rspec options]]
+    $ bundle exec snap-ci-parallel-test [options] [files or directories] [-- [Test::Unit or MiniTest options]]
 
 
 ## Setup for Rails
 
-Ensure that 'parallel_tests' is present in your development group
+Ensure that 'snap_ci-parallel_tests' is present in your development group
 
 ```ruby
 # Gemfile
-gem "parallel_tests", :group => :development
+gem "snap_ci-parallel_tests", :group => :development
 ```
 
 ### Run
 
-    $ bundle exec snap_ci-parallel_rspec [options] [files or directories] [-- [rspec options]]
-    $ bundle exec snap_ci-parallel_test [options] [files or directories] [-- [Test::Unit or MiniTest options]]
+    $ bundle exec snap-ci-parallel-rspec [options] [files or directories] [-- [rspec options]]
+    $ bundle exec snap-ci-parallel-test [options] [files or directories] [-- [Test::Unit or MiniTest options]]
 
 Alternatively -
 
@@ -53,7 +53,7 @@ $ bundle exec rake -T snap-parallel          # to list all tasks
 
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/snap_ci-parallel_tests/fork )
+1. Fork it ( https://github.com/[my-github-username]/parallel-tests/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/bin/snap-ci-parallel-partition

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.expand_path("../../lib", __FILE__)
+require 'snap_ci/parallel_tests'
+require 'snap_ci/parallel_tests/cli'
+require 'snap_ci/parallel_tests/partition/runner'
+
+SnapCI::ParallelTests::CLI.new(SnapCI::ParallelTests::Partition::Runner.new, ARGV).run

data/lib/snap_ci/parallel_tests/cli.rb

@@ -73,6 +73,7 @@ module SnapCI
       end
 
       def report_number_of_tests(tests, total_workers)
+        return unless runner.report_test_partitioning?
         name = runner.test_file_name
         num_tests = tests.size
         $stderr.puts "#{total_workers} workers for #{num_tests} #{name}s, ~ #{(num_tests.to_f/total_workers).ceil} #{name}s per process"
@@ -84,20 +85,35 @@ module SnapCI
         options[:group_by] = :filename
 
         parser = OptionParser.new do |opts|
-          runner.cli_helper.render_header(opts)
+          runner.cli_helper.render_header(opts, options)
           opts.separator 'supported options:'
 
-          runner.cli_helper.render_options(opts)
+          runner.cli_helper.render_options(opts, options)
 
           opts.on('-g', '--group-by TYPE', <<-TEXT) do |type|
 group tests by:
           filename - order of finding files(default)
           filesize - by size of the file
             TEXT
-            raise unless %w(name filesize).include?(type)
+
+            allowed_group_values = %w(filename filesize)
+            raise "Group option only supports #{allowed_group_values.join(', ')}. You passed #{type}" unless allowed_group_values.include?(type)
+
             options[:group_by] = type.to_sym
           end
 
+          opts.on('-d', '--distribution TYPE', <<-TEXT) do |type|
+after grouping tests, distribute tests across workers by:
+          roundrobin - distribute files one at a time to each worker (default)
+          chunk      - slice tests
+            TEXT
+
+            allowed_distribution_values = %w(roundrobin chunk)
+            raise "Distribution option only supports #{allowed_distribution_values.join(', ')}. You passed #{type}" unless allowed_distribution_values.include?(type)
+
+            options[:distribution] = type.to_sym
+          end
+
           opts.on('-v', '--version', 'Show Version') do
             puts "SnapCI Parallel Tests v#{SnapCI::ParallelTests::VERSION}"
             exit
@@ -112,7 +128,7 @@ group tests by:
             exit
           end
 
-          runner.cli_helper.render_footer(opts)
+          runner.cli_helper.render_footer(opts, options)
         end
 
         parser.parse!(argv)

data/lib/snap_ci/parallel_tests/grouper.rb

@@ -9,6 +9,27 @@ module SnapCI
         def group_by_filesize(things)
           things.sort { |a, b| File.size(a) <=> File.size(b) }
         end
+
+        def distribute_by_round_robin(things, total_workers, current_worker_index)
+          result = []
+
+          # pick up things on a round-robin basis to distribute them evenly
+          index = current_worker_index - 1
+          while index <= things.count do
+            result << things[index]
+            index += total_workers
+          end
+          result.compact!
+
+          result
+        end
+
+        def distribute_by_chunk(things, total_workers, current_worker_index)
+          thing_count = things.count
+          specs_per_worker = (thing_count.to_f/total_workers).ceil
+
+          things.each_slice(specs_per_worker).to_a[current_worker_index-1] || []
+        end
       end
     end
   end

data/lib/snap_ci/parallel_tests/partition/cli_helper.rb

@@ -0,0 +1,35 @@
+module SnapCI
+  module ParallelTests
+    module Partition
+      module CLIHelper
+        def render_header(optparser, options)
+          optparser.banner = <<BANNER
+Partition a list of files/directories and print them
+
+Usage: #{optparser.program_name} [options] [files or directories]
+
+Example: #{optparser.program_name} test/models test/controllers/foo_controller_test.rb
+
+Can typically be used as -
+
+  $ your-test-runner $(snap-ci-parallel-partition location/of/test/files)
+
+BANNER
+        end
+
+        def render_options(optparser, options)
+          optparser.on('-p', '--pattern [PATTERN]', 'only find files matching this pattern') do |pattern|
+            options[:pattern] = /#{pattern}/
+          end
+        end
+
+        def render_footer(optparser, options)
+          optparser.separator ''
+        end
+
+        extend CLIHelper
+      end #CLIHelper
+
+    end #RSpec
+  end #ParallelTests
+end #SnapCI

data/lib/snap_ci/parallel_tests/partition/runner.rb

@@ -0,0 +1,33 @@
+require 'shellwords'
+require 'snap_ci/parallel_tests/partition/cli_helper'
+
+module SnapCI
+  module ParallelTests
+    module Partition
+      class Runner
+
+        def execute(test_files, options)
+          $stdout.puts(test_files.collect {|f| f.dump }.join(' '))
+        end
+
+        def report_test_partitioning?
+          false
+        end
+
+        def cli_helper
+          CLIHelper
+        end
+
+        def test_suffix
+          //
+        end
+
+        def test_file_name
+          'file'
+        end
+
+      end #Runner
+
+    end #Partition
+  end #ParallelTests
+end #SnapCI

data/lib/snap_ci/parallel_tests/rspec/cli_helper.rb

@@ -2,22 +2,22 @@ module SnapCI
   module ParallelTests
     module RSpec
       module CLIHelper
-        def render_header(opts)
-          opts.banner = <<BANNER
-Usage: #{opts.program_name} [options] [files or directories] [-- [rspec options]]
+        def render_header(optparser, options)
+          optparser.banner = <<BANNER
+Usage: #{optparser.program_name} [options] [files or directories] [-- [rspec options]]
 
-Example: #{opts.program_name} spec/models spec/controllers/foo_controller_spec.rb -- --format documentation --fail-fast
+Example: #{optparser.program_name} spec/models spec/controllers/foo_controller_spec.rb -- --format documentation --fail-fast
 BANNER
         end
 
-        def render_options(opts)
-          opts.on('-p', '--pattern [PATTERN]', 'run tests matching this pattern') do |pattern|
+        def render_options(optparser, options)
+          optparser.on('-p', '--pattern [PATTERN]', 'run tests matching this pattern') do |pattern|
             options[:pattern] = /#{pattern}/
           end
         end
 
-        def render_footer(opts)
-          opts.separator "\nRun `rspec --help' for supported rspec options."
+        def render_footer(optparser, options)
+          optparser.separator "\nRun `rspec --help' for supported rspec options."
         end
 
         extend CLIHelper

data/lib/snap_ci/parallel_tests/rspec/runner.rb

@@ -19,6 +19,10 @@ module SnapCI
           exec((options[:env] || {}), cmd)
         end
 
+        def report_test_partitioning?
+          true
+        end
+
         def cli_helper
           CLIHelper
         end

data/lib/snap_ci/parallel_tests/test/cli_helper.rb

@@ -2,27 +2,27 @@ module SnapCI
   module ParallelTests
     module Test
       module CLIHelper
-        def render_header(opts)
-          opts.banner = <<BANNER
-Usage: #{opts.program_name} [options] [files or directories] [-- [Test::Unit or MiniTest options]]
+        def render_header(optparser, options)
+          optparser.banner = <<BANNER
+Usage: #{optparser.program_name} [options] [files or directories] [-- [Test::Unit or MiniTest options]]
 
-Example: #{opts.program_name} test/models test/controllers/foo_controller_test.rb -- --verbose --seed 10
+Example: #{optparser.program_name} test/models test/controllers/foo_controller_test.rb -- --verbose --seed 10
 BANNER
         end
 
-        def render_options(opts)
-          opts.on('-p', '--pattern [PATTERN]', 'run tests matching this pattern') do |pattern|
+        def render_options(optparser, options)
+          optparser.on('-p', '--pattern [PATTERN]', 'run tests matching this pattern') do |pattern|
             options[:pattern] = /#{pattern}/
           end
         end
 
-        def render_footer(opts)
-          opts.separator "\nRun `ruby -r test/test_helper -e1 --help' for supported Test::Unit or MiniTest options."
+        def render_footer(optparser, options)
+          optparser.separator "\nRun `ruby -r test/test_helper -e1 --help' for supported Test::Unit or MiniTest options."
         end
 
         extend CLIHelper
       end #CLIHelper
 
-    end #RSpec
+    end #Test
   end #ParallelTests
 end #SnapCI

data/lib/snap_ci/parallel_tests/test/runner.rb

@@ -16,6 +16,10 @@ module SnapCI
           exec(cmd)
         end
 
+        def report_test_partitioning?
+          true
+        end
+
         def cli_helper
           CLIHelper
         end
@@ -28,26 +32,8 @@ module SnapCI
           'test'
         end
 
-        private
-
-        def output_of(cmd)
-          `#{cmd}`
-        end
-
-        def rspec_2_color
-          '--color --tty' if $stdout.tty?
-        end
-
-        def rspec_1_color
-          if $stdout.tty?
-            { 'RSPEC_COLOR' => '1' }
-          else
-            {}
-          end
-        end
-
       end #Runner
 
-    end #RSpec
+    end #Test
   end #ParallelTests
 end #SnapCI

data/lib/snap_ci/parallel_tests/version.rb

@@ -1,5 +1,5 @@
 module SnapCI
   module ParallelTests
-    VERSION = '0.0.1'
+    VERSION = '0.0.2'
   end
 end

data/lib/snap_ci/parallel_tests.rb

@@ -10,29 +10,23 @@ module SnapCI
     #
     # ==== Options
     #   +things+ - the things to partition
-    #   +total_workers+ - the total number of workers, defaults to ParallelTests.total_workers
-    #   +current_worker_index+ - the current worker index (1 based, NOT 0 based), defaults to ParallelTests.worker_index
-    #   +group_by+ - either :filename or :filesize (defaults to :filename). Determines how files are sorted before being partitioned
+    #   +total_workers+ - the total number of workers, defaults to ParallelTests.total_workers.
+    #   +current_worker_index+ - the current worker index (1 based, NOT 0 based), defaults to ParallelTests.worker_index.
+    #   +group_by+ - either :filename or :filesize (defaults to :filename). Determines how files are sorted before being distributed into partitions/
+    #   +distribution+ - either :round_robin or :chunk (defaults to :roundrobin). Determines how files are distributed across workers, after they are sorted.
     def partition(options={})
       things = options[:things]
       total_workers = options[:total_workers] || ParallelTests.total_workers
       current_worker_index = options[:current_worker_index] || ParallelTests.worker_index
       group_by = options[:group_by] || :filename
+      distribution = options[:distribution] || :round_robin
 
       return [] if things.nil? || things.empty?
       things = Grouper.send("group_by_#{group_by}", things)
 
-      result = []
+      things = Grouper.send("distribute_by_#{distribution}", things, total_workers, current_worker_index)
 
-      # pick up things on a round-robin basis to distribute them evenly
-      index = current_worker_index - 1
-      while index <= things.count do
-        result << things[index]
-        index += total_workers
-      end
-      result.compact!
-
-      result
+      things
     end
 
     def total_workers

data/snap_ci-parallel_tests.gemspec

@@ -18,6 +18,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 1.4"
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/spec/snap_ci/parallel_tests_spec.rb

@@ -3,41 +3,104 @@ require 'spec_helper'
 describe SnapCI::ParallelTests do
   include SnapCI::ParallelTests
 
-  it 'should create uniform partitions of things based on count and index' do
-    things = (1..10).to_a
+  describe 'default (round robin) distribution behavior, when distribution option is not passed' do
 
-    partition1 = partition(things: things, total_workers: 3, current_worker_index: 1)
-    partition2 = partition(things: things, total_workers: 3, current_worker_index: 2)
-    partition3 = partition(things: things, total_workers: 3, current_worker_index: 3)
+    it 'should create uniform partitions of things based on count and index' do
+      things = (1..10).to_a
 
-    expect(partition1 + partition2 + partition3).to contain_exactly(*things)
+      partition1 = partition(things: things, total_workers: 3, current_worker_index: 1)
+      partition2 = partition(things: things, total_workers: 3, current_worker_index: 2)
+      partition3 = partition(things: things, total_workers: 3, current_worker_index: 3)
+
+      expect(partition1 + partition2 + partition3).to contain_exactly(*things)
+
+      expect(partition1).to eq([1, 4, 7, 10])
+      expect(partition2).to eq([2, 5, 8])
+      expect(partition3).to eq([3, 6, 9])
+    end
+
+    it 'should create empty partitions if things are empty' do
+      things = []
+
+      partition1 = partition(things: things, total_workers: 3, current_worker_index: 1)
+      partition2 = partition(things: things, total_workers: 3, current_worker_index: 2)
+      partition3 = partition(things: things, total_workers: 3, current_worker_index: 3)
+
+      expect(partition1).to eq([])
+      expect(partition2).to eq([])
+      expect(partition3).to eq([])
+    end
+
+    it 'should create empty partitions at end if partitions are more than number of things' do
+      things = [1, 2]
+
+      partition1 = partition(things: things, total_workers: 3, current_worker_index: 1)
+      partition2 = partition(things: things, total_workers: 3, current_worker_index: 2)
+      partition3 = partition(things: things, total_workers: 3, current_worker_index: 3)
+
+      expect(partition1).to eq([1])
+      expect(partition2).to eq([2])
+      expect(partition3).to eq([])
+    end
 
-    expect(partition1).to eq([1, 4, 7, 10])
-    expect(partition2).to eq([2, 5, 8])
-    expect(partition3).to eq([3, 6, 9])
   end
 
-  it 'should create empty partitions if things are empty' do
-    things = []
+  describe 'round robin distribution, when distribution option is not passed' do
+    it 'should create uniform partitions of things based on count and index' do
+      things = (1..10).to_a
 
-    partition1 = partition(things: things, total_workers: 3, current_worker_index: 1)
-    partition2 = partition(things: things, total_workers: 3, current_worker_index: 2)
-    partition3 = partition(things: things, total_workers: 3, current_worker_index: 3)
+      partition1 = partition(things: things, total_workers: 3, current_worker_index: 1, distribution: :round_robin)
+      partition2 = partition(things: things, total_workers: 3, current_worker_index: 2, distribution: :round_robin)
+      partition3 = partition(things: things, total_workers: 3, current_worker_index: 3, distribution: :round_robin)
 
-    expect(partition1).to eq([])
-    expect(partition2).to eq([])
-    expect(partition3).to eq([])
+      expect(partition1 + partition2 + partition3).to contain_exactly(*things)
+
+      expect(partition1).to eq([1, 4, 7, 10])
+      expect(partition2).to eq([2, 5, 8])
+      expect(partition3).to eq([3, 6, 9])
+    end
   end
 
-  it 'should create empty partitions at end if partitions are more than number of things' do
-    things = [1, 2]
+  describe 'chunk distribution' do
+
+    it 'should create uniform partitions of things based on count and index' do
+      things = (1..10).to_a
 
-    partition1 = partition(things: things, total_workers: 3, current_worker_index: 1)
-    partition2 = partition(things: things, total_workers: 3, current_worker_index: 2)
-    partition3 = partition(things: things, total_workers: 3, current_worker_index: 3)
+      partition1 = partition(things: things, total_workers: 3, current_worker_index: 1, distribution: :chunk)
+      partition2 = partition(things: things, total_workers: 3, current_worker_index: 2, distribution: :chunk)
+      partition3 = partition(things: things, total_workers: 3, current_worker_index: 3, distribution: :chunk)
+
+      expect(partition1 + partition2 + partition3).to contain_exactly(*things)
+
+      expect(partition1).to eq([1, 2, 3, 4])
+      expect(partition2).to eq([5, 6, 7, 8])
+      expect(partition3).to eq([9, 10])
+    end
+
+    it 'should create empty partitions if things are empty' do
+      things = []
+
+      partition1 = partition(things: things, total_workers: 3, current_worker_index: 1, distribution: :chunk)
+      partition2 = partition(things: things, total_workers: 3, current_worker_index: 2, distribution: :chunk)
+      partition3 = partition(things: things, total_workers: 3, current_worker_index: 3, distribution: :chunk)
+
+      expect(partition1).to eq([])
+      expect(partition2).to eq([])
+      expect(partition3).to eq([])
+    end
+
+    it 'should create empty partitions at end if partitions are more than number of things' do
+      things = [1, 2]
+
+      partition1 = partition(things: things, total_workers: 3, current_worker_index: 1, distribution: :chunk)
+      partition2 = partition(things: things, total_workers: 3, current_worker_index: 2, distribution: :chunk)
+      partition3 = partition(things: things, total_workers: 3, current_worker_index: 3, distribution: :chunk)
+
+      expect(partition1).to eq([1])
+      expect(partition2).to eq([2])
+      expect(partition3).to eq([])
+    end
 
-    expect(partition1).to eq([1])
-    expect(partition2).to eq([2])
-    expect(partition3).to eq([])
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: snap_ci-parallel_tests
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Ketan Padegaonkar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-10 00:00:00.000000000 Z
+date: 2014-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -42,8 +42,9 @@ description: Run Test::Unit / RSpec in parallel across multiple workers on Snap
 email:
 - KetanPadegaonkar@gmail.com
 executables:
-- snap_ci-parallel_rspec
-- snap_ci-parallel_test
+- snap-ci-parallel-partition
+- snap-ci-parallel-rspec
+- snap-ci-parallel-test
 extensions: []
 extra_rdoc_files: []
 files:
@@ -56,11 +57,14 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/snap_ci-parallel_rspec
-- bin/snap_ci-parallel_test
+- bin/snap-ci-parallel-partition
+- bin/snap-ci-parallel-rspec
+- bin/snap-ci-parallel-test
 - lib/snap_ci/parallel_tests.rb
 - lib/snap_ci/parallel_tests/cli.rb
 - lib/snap_ci/parallel_tests/grouper.rb
+- lib/snap_ci/parallel_tests/partition/cli_helper.rb
+- lib/snap_ci/parallel_tests/partition/runner.rb
 - lib/snap_ci/parallel_tests/railtie.rb
 - lib/snap_ci/parallel_tests/rspec/cli_helper.rb
 - lib/snap_ci/parallel_tests/rspec/runner.rb
@@ -91,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.4
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Run Test::Unit / RSpec in parallel across multiple workers on Snap CI