paraduct 0.0.1.beta4 → 0.0.1.beta5

data/README.md

@@ -65,6 +65,7 @@ variables:
   name2:
     - value2a
     - value2b
+max_threads: 4
 ```
 
 ### script
@@ -88,6 +89,9 @@ value1a | value2b | NAME1_value1a_NAME2_value2b   | tmp/paraduct_workspace/NAME1
 value1b | value2a | NAME1_value1b_NAME2_value2a   | tmp/paraduct_workspace/NAME1_value1b_NAME2_value2a
 value1b | value2b | NAME1_value1b_NAME2_value2b   | tmp/paraduct_workspace/NAME1_value1b_NAME2_value2b
 
+### max_threads
+maximum concurrent execution number of jobs
+
 ## Contributing
 
 1. Fork it ( https://github.com/sue445/paraduct/fork )

data/lib/paraduct.rb

@@ -1,20 +1,25 @@
 require "active_support"
 require "active_support/deprecation"
 require "active_support/core_ext"
-require "paraduct/version"
-require "paraduct/configuration"
-require "paraduct/variable_converter"
-require "paraduct/runner"
-require "paraduct/parallel_runner"
-require "paraduct/errors"
-require "paraduct/test_response"
 require "pathname"
 
 module Paraduct
+  autoload :Configuration    , 'paraduct/configuration'
+  autoload :Errors           , 'paraduct/errors'
+  autoload :ParallelRunner   , 'paraduct/parallel_runner'
+  autoload :Runner           , 'paraduct/runner'
+  autoload :TestResponse     , 'paraduct/test_response'
+  autoload :VariableConverter, 'paraduct/variable_converter'
+  autoload :Version          , 'paraduct/version'
+
   class << self
     def configuration
       Paraduct::Configuration.instance
     end
     alias :config :configuration
+
+    def logger
+      @logger ||= ActiveSupport::Logger.new(STDOUT)
+    end
   end
 end

data/lib/paraduct/cli.rb

@@ -15,8 +15,8 @@ module Paraduct
 
       product_variables = Paraduct::VariableConverter.product(variables)
       test_response = Paraduct::ParallelRunner.perform_all(script, product_variables)
-      puts test_response.detail_message
-      raise Paraduct::TestFailureError if test_response.failure?
+      Paraduct.logger.info test_response.detail_message
+      raise Paraduct::Errors::TestFailureError if test_response.failure?
     end
 
     desc "generate", "generate .paraduct.yml"

data/lib/paraduct/configuration.rb

@@ -20,6 +20,11 @@ module Paraduct
       @config["script"]
     end
 
+    # @return [Integer]
+    def max_threads
+      @config["max_threads"] || 4
+    end
+
     # @return [Pathname]
     def work_dir
       _work_dir = @config["work_dir"] || "tmp/paraduct_workspace"

data/lib/paraduct/errors.rb

@@ -1,14 +1,16 @@
 module Paraduct
-  class TestFailureError < StandardError; end
+  module Errors
+    class TestFailureError < StandardError; end
 
-  class ProcessError < StandardError
-    attr_reader :status
+    class ProcessError < StandardError
+      attr_reader :status
 
-    # @param message [String] stdout and stderr
-    # @param status  [Process::Status]
-    def initialize(message, status)
-      super(message)
-      @status = status
+      # @param message [String] stdout and stderr
+      # @param status  [Process::Status]
+      def initialize(message, status)
+        super(message)
+        @status = status
+      end
     end
   end
 end

data/lib/paraduct/parallel_runner.rb

@@ -1,58 +1,65 @@
 module Paraduct
+  require "thread/pool"
+
   class ParallelRunner
     # run script with arguments
     # @param script            [String, Array<String>] script file, script(s)
     # @param product_variables [Array<Hash{String => String}>]
     # @return [Paraduct::TestResponse]
     def self.perform_all(script, product_variables)
-      threads = []
       test_response = Paraduct::TestResponse.new
       base_job_dir = Paraduct.config.work_dir
       FileUtils.mkdir_p(base_job_dir) unless base_job_dir.exist?
 
-      puts <<-EOS
+      Paraduct.logger.info <<-EOS
 ======================================================
 START matrix test
       EOS
       product_variables.each do |params|
-        puts "params: #{params}"
+        Paraduct.logger.info "params: #{params}"
       end
 
-      product_variables.each do |params|
-        runner = Paraduct::Runner.new(
-          script:       script,
-          params:       params,
-          base_job_dir: base_job_dir,
-        )
-        threads << Thread.new(runner) do |_runner|
-          _runner.setup_dir
-          begin
-            stdout = _runner.perform
-            successful = true
-          rescue Paraduct::ProcessError => e
-            stdout = e.message
-            successful = false
-          end
+      pool = Thread.pool(Paraduct.config.max_threads)
+      begin
+        product_variables.each do |params|
+          pool.process do
+            runner = Paraduct::Runner.new(
+              script:       script,
+              params:       params,
+              base_job_dir: base_job_dir,
+            )
+
+            runner.setup_dir
+            begin
+              stdout = runner.perform
+              successful = true
+            rescue Paraduct::Errors::ProcessError => e
+              stdout = e.message
+              successful = false
+            end
 
-          puts <<-EOS
+            Paraduct.logger.info <<-EOS
 ======================================================
-params:   #{_runner.formatted_params}
-job_name: #{_runner.job_name}
-job_dir:  #{_runner.job_dir}
-
-#{stdout}
-          EOS
-
-          test_response.jobs_push(
-            job_name:         _runner.job_name,
-            params:           _runner.params,
-            formatted_params: _runner.formatted_params,
-            successful:       successful,
-            stdout:           stdout,
-          )
+params:   #{runner.formatted_params}
+job_name: #{runner.job_name}
+job_dir:  #{runner.job_dir}
+
+            #{stdout}
+            EOS
+
+            test_response.jobs_push(
+              job_name:         runner.job_name,
+              params:           runner.params,
+              formatted_params: runner.formatted_params,
+              successful:       successful,
+              stdout:           stdout,
+            )
+          end
         end
+
+      ensure
+        pool.shutdown
       end
-      threads.map(&:join)
 
       test_response
     end

data/lib/paraduct/runner.rb

@@ -22,7 +22,7 @@ module Paraduct
 
     # run script with params
     # @return [String] stdout
-    # @raise [Paraduct::ProcessError] command exited error status
+    # @raise [Paraduct::Errors::ProcessError] command exited error status
     def perform
       variable_string = key_capitalized_params.map{ |key, value| %(export #{key}="#{value}";) }.join(" ")
 
@@ -72,7 +72,7 @@ module Paraduct
     private
     def run_command(command)
       stdout, stderr, status = Open3.capture3(command)
-      raise Paraduct::ProcessError.new("#{stdout}\n#{stderr}", status) unless status.success?
+      raise Paraduct::Errors::ProcessError.new("#{stdout}\n#{stderr}", status) unless status.success?
       stdout
     end
   end

data/lib/paraduct/templates/.paraduct.yml.tt

@@ -8,3 +8,4 @@ variables:
   name2:
     - value2a
     - value2b
+max_threads: 4

data/lib/paraduct/version.rb

@@ -1,3 +1,3 @@
 module Paraduct
-  VERSION = "0.0.1.beta4"
+  VERSION = "0.0.1.beta5"
 end

data/paraduct.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activesupport"
   spec.add_dependency "thor"
+  spec.add_dependency "thread"
 
   spec.add_development_dependency "bundler", ">= 1.5"
   spec.add_development_dependency "codeclimate-test-reporter"

data/spec/.paraduct.yml

@@ -12,3 +12,4 @@ variables:
     - 3.2.0
     - 4.0.0
     - 4.1.0
+max_threads: 4

data/spec/paraduct/cli_spec.rb

@@ -51,7 +51,7 @@ describe Paraduct::CLI do
         allow(Paraduct.config).to receive(:variables){ { status: [0, 1] } }
       end
 
-      it { expect{ subject }.to raise_error Paraduct::TestFailureError }
+      it { expect{ subject }.to raise_error Paraduct::Errors::TestFailureError }
     end
   end
 

data/spec/paraduct/runner_spec.rb

@@ -44,7 +44,7 @@ DATABASE=mysql
           EOS
         end
 
-        it { expect{ subject }.to raise_error(Paraduct::ProcessError, stdout) }
+        it { expect{ subject }.to raise_error(Paraduct::Errors::ProcessError, stdout) }
       end
     end
 

data/spec/spec_helper.rb

@@ -119,6 +119,6 @@ RSpec.configure do |config|
 
   config.before do
     # quiet
-    allow(Object).to receive(:puts)
+    allow(Paraduct.logger).to receive(:info)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: paraduct
 version: !ruby/object:Gem::Version
-  version: 0.0.1.beta4
+  version: 0.0.1.beta5
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-19 00:00:00.000000000 Z
+date: 2014-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -313,7 +329,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3632247312778024801
+      hash: 981450971291940130
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: