parallel_minion 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ec80ef2321aa78ba6f8b6dc7b16452f1ccb15a9b
-  data.tar.gz: f3a394698da183ce6e64c8137ac7cf50cae98943
+SHA256:
+  metadata.gz: 9ca9bfc8f399bcbb7cf2d90c3bb71c30fb160b23890c53ca4ca7add4a5b12fdb
+  data.tar.gz: a196538e92034bce7281060fcc6f22db09d3c26c49c5b6175f2471d2015aea7b
 SHA512:
-  metadata.gz: 946b4306251796613bf5a44b89d32016aa0a9133048caaaa698206151aeb5e9dc4684e377230d767b2fb8bcc2f952a83c18c140f96bcb7e5c44fdee10f88c16c
-  data.tar.gz: 1f52084ea14e186ff3796f8276fbc879bb99ada6d0b05fead4513b26a4611a471064c2ebbbc0cdb27ced7e1e9a41135674083436b2737108a54b921f640fb1fc
+  metadata.gz: 6c8f413c6fda9af1dcb94efdc4e16bf40183359f942410f38a7c57884def7ca53aee16d51c824b24c499037288ed56a5458b329bb92e87f16f8cf73c9922c15a
+  data.tar.gz: 44697598c5916f29a6b02bc15cf26fcd5871de81a13cc7426192d4e49ea11ed21d0fa03d9085f44daa0be0a2f565f9cf3978e3ffe84f4487910e5400870a16c6

data/README.md

@@ -1,7 +1,7 @@
 parallel_minion [![Build Status](https://secure.travis-ci.org/reidmorrison/parallel_minion.png?branch=master)](http://travis-ci.org/reidmorrison/parallel_minion)
 ===============
 
-Pragmatic approach to parallel processing in Ruby
+Wrap Ruby code with a minion so that it is run on a parallel thread.
 
 ## Description
 

data/Rakefile

@@ -9,7 +9,7 @@ task :gem do
   system 'gem build parallel_minion.gemspec'
 end
 
-task :publish => :gem do
+task publish: :gem do
   system "git tag -a v#{ParallelMinion::VERSION} -m 'Tagging #{ParallelMinion::VERSION}'"
   system 'git push --tags'
   system "gem push parallel_minion-#{ParallelMinion::VERSION}.gem"
@@ -23,7 +23,7 @@ Rake::TestTask.new(:test) do |t|
 end
 
 # By default run tests against all appraisals
-if !ENV["APPRAISAL_INITIALIZED"] && !ENV["TRAVIS"]
+if !ENV['APPRAISAL_INITIALIZED'] && !ENV['TRAVIS']
   require 'appraisal'
   task default: :appraisal
 else

data/lib/parallel_minion.rb

@@ -1,4 +1,3 @@
-require 'thread'
 require 'semantic_logger'
 
 module ParallelMinion

data/lib/parallel_minion/minion.rb

@@ -22,7 +22,7 @@ module ParallelMinion
     # Metrics [String]
     attr_reader :metric, :wait_metric
 
-    attr_reader :on_timeout, :log_exception, :start_time
+    attr_reader :on_timeout, :log_exception, :start_time, :on_exception_level
 
     # Give an infinite amount of time to wait for a Minion to complete a task
     INFINITE = 0
@@ -40,8 +40,8 @@ module ParallelMinion
     # - Production:
     #     Batch processing in Rocket Job where throughput is more important than latency.
     #       http://rocketjob.io
-    def self.enabled=(enabled)
-      @enabled = enabled
+    class << self
+      attr_writer :enabled
     end
 
     # Returns whether minions are enabled to run in their own threads
@@ -54,8 +54,8 @@ module ParallelMinion
     #
     # Example:
     #   ...
-    def self.scoped_classes
-      @scoped_classes
+    class << self
+      attr_reader :scoped_classes
     end
 
     def self.scoped_classes=(scoped_classes)
@@ -73,8 +73,8 @@ module ParallelMinion
       @started_log_level = level
     end
 
-    def self.started_log_level
-      @started_log_level
+    class << self
+      attr_reader :started_log_level
     end
 
     # Change the log level for the Completed log message.
@@ -88,8 +88,8 @@ module ParallelMinion
       @completed_log_level = level
     end
 
-    def self.completed_log_level
-      @completed_log_level
+    class << self
+      attr_reader :completed_log_level
     end
 
     self.started_log_level   = :info
@@ -177,6 +177,10 @@ module ParallelMinion
     #        Any unhandled exception raised in the block will not be logged
     #      Default: :partial
     #
+    #   :on_exception_level [:trace | :debug | :info | :warn | :error | :fatal]
+    #     Override the log level only when an exception occurs.
+    #     Default: ParallelMinion::Minion.completed_log_level
+    #
     #   :enabled [Boolean]
     #     Override the global setting: `ParallelMinion::Minion.enabled?` for this minion instance.
     #
@@ -193,25 +197,53 @@ module ParallelMinion
     #     thread.pool.enabled=true
     #
     # Example:
-    #   ParallelMinion::Minion.new(10.days.ago, description: 'Doing something else in parallel', timeout: 1000) do |date|
+    #   ParallelMinion::Minion.new(
+    #     10.days.ago,
+    #     description: 'Doing something else in parallel',
+    #     timeout:     1000
+    #   ) do |date|
     #     MyTable.where('created_at <= ?', date).count
     #   end
-    def initialize(*arguments, description: 'Minion', metric: nil, log_exception: nil, enabled: self.class.enabled?, timeout: INFINITE, on_timeout: nil, wait_metric: nil, &block)
+    #
+    # Example, when the result is being ignored, log full exception as an error:
+    #   ParallelMinion::Minion.new(
+    #     customer,
+    #     description:        "We don't care about the result",
+    #     log_exception:      :full,
+    #     on_exception_level: :error
+    #   ) do |customer|
+    #     customer.save!
+    #   end
+    def initialize(*arguments,
+                   description: 'Minion',
+                   metric: nil,
+                   log_exception: nil,
+                   on_exception_level: self.class.completed_log_level,
+                   enabled: self.class.enabled?,
+                   timeout: INFINITE,
+                   on_timeout: nil,
+                   wait_metric: nil,
+                   &block)
       raise 'Missing mandatory block that Minion must perform' unless block
-      @start_time    = Time.now
-      @exception     = nil
-      @arguments     = arguments
-      @timeout       = timeout.to_f
-      @description   = description.to_s
-      @metric        = metric
-      @log_exception = log_exception
-      @enabled       = enabled
-      @on_timeout    = on_timeout
-
-      @wait_metric   = (wait_metric || "#{metric}/wait") if @metric
+      @start_time         = Time.now
+      @exception          = nil
+      @arguments          = arguments
+      @timeout            = timeout.to_f
+      @description        = description.to_s
+      @metric             = metric
+      @log_exception      = log_exception
+      @on_exception_level = on_exception_level
+      @enabled            = enabled
+      @on_timeout         = on_timeout
+
+      @wait_metric        = (wait_metric || "#{metric}/wait") if @metric
 
       # When minion is disabled it is obvious in the logs since the name will now be 'Inline' instead of 'Minion'
-      self.logger    = SemanticLogger['Inline'] unless @enabled
+      unless @enabled
+        l           = self.class.logger.dup
+        l.name      = 'Inline'
+        self.logger = l
+      end
 
       @enabled ? run(&block) : run_inline(&block)
     end
@@ -225,7 +257,12 @@ module ParallelMinion
       # Return nil if Minion is still working and has time left to finish
       if working?
         ms = time_left
-        logger.measure(self.class.completed_log_level, "Waited for Minion to complete: #{description}", min_duration: 0.01, metric: wait_metric) do
+        logger.measure(
+          self.class.completed_log_level,
+          "Waited for Minion to complete: #{description}",
+          min_duration: 0.01,
+          metric:       wait_metric
+        ) do
           if @thread.join(ms.nil? ? nil : ms / 1000).nil?
             @thread.raise(@on_timeout.new("Minion: #{description} timed out")) if @on_timeout
             logger.warn("Timed out waiting for: #{description}")
@@ -257,7 +294,7 @@ module ParallelMinion
     # Returns 0 if no time is left
     # Returns nil if their is no time limit. I.e. :timeout was set to Minion::INFINITE (infinite time left)
     def time_left
-      return nil if (timeout == 0) || (timeout == -1)
+      return nil if timeout.zero? || (timeout == -1)
       duration = timeout - (Time.now - start_time) * 1000
       duration <= 0 ? 0 : duration
     end
@@ -272,32 +309,40 @@ module ParallelMinion
     if defined?(ActiveRecord)
       if ActiveRecord::VERSION::MAJOR >= 4
         def self.current_scopes
-          scoped_classes.collect { |klass| klass.all }
+          scoped_classes.collect(&:all)
         end
       else
         def self.current_scopes
-          scoped_classes.collect { |klass| klass.scoped }
+          scoped_classes.collect(&:scoped)
         end
       end
     end
 
     private
 
+    # rubocop:disable Lint/RescueException
+
     # Run the supplied block of code in the current thread.
     # Useful for debugging, testing, and when running in batch environments.
     def run_inline(&block)
-      begin
-        logger.public_send(self.class.started_log_level, "Started #{@description}")
-        logger.measure(self.class.completed_log_level, "Completed #{@description}", log_exception: @log_exception, metric: metric) do
-          @result = instance_exec(*arguments, &block)
-        end
-      rescue Exception => exc
-        @exception = exc
-      ensure
-        @duration = Time.now - start_time
+      logger.public_send(self.class.started_log_level, "Started #{description}")
+      logger.measure(
+        self.class.completed_log_level,
+        "Completed #{description}",
+        log_exception:      log_exception,
+        on_exception_level: on_exception_level,
+        metric:             metric
+      ) do
+        @result = instance_exec(*arguments, &block)
       end
+    rescue Exception => exc
+      @exception = exc
+    ensure
+      @duration = Time.now - start_time
     end
 
+    # rubocop:enable Lint/RescueException
+
     def run(&block)
       # Capture tags from current thread
       tags = SemanticLogger.tags
@@ -316,9 +361,17 @@ module ParallelMinion
         SemanticLogger.tagged(*tags) do
           SemanticLogger.named_tagged(named_tags) do
             logger.public_send(self.class.started_log_level, "Started #{description}")
+            # rubocop:disable Lint/RescueException
             begin
-              proc = Proc.new { run_in_scope(scopes, &block) }
-              logger.measure(self.class.completed_log_level, "Completed #{description}", log_exception: log_exception, metric: metric, &proc)
+              proc = proc { run_in_scope(scopes, &block) }
+              logger.measure(
+                self.class.completed_log_level,
+                "Completed #{description}",
+                log_exception:      log_exception,
+                on_exception_level: on_exception_level,
+                metric:             metric,
+                &proc
+              )
             rescue Exception => exc
               @exception = exc
               nil
@@ -327,6 +380,7 @@ module ParallelMinion
               # Return any database connections used by this thread back to the pool
               ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
             end
+            # rubocop:enable Lint/RescueException
           end
         end
       end
@@ -338,12 +392,11 @@ module ParallelMinion
       else
         # Use the captured scope when running the block.
         # Each Class to scope requires passing a block to .scoping.
-        proc  = Proc.new { instance_exec(*@arguments, &block) }
+        proc  = proc { instance_exec(*@arguments, &block) }
         first = scopes.shift
-        scopes.each { |scope| proc = Proc.new { scope.scoping(&proc) } }
+        scopes.each { |scope| proc = proc { scope.scoping(&proc) } }
         @result = first.scoping(&proc)
       end
     end
-
   end
 end

data/lib/parallel_minion/railtie.rb

@@ -26,6 +26,5 @@ module ParallelMinion #:nodoc:
     #     end
     #   end
     config.parallel_minion = ::ParallelMinion::Minion
-
   end
 end

data/lib/parallel_minion/version.rb

@@ -1,3 +1,3 @@
-module ParallelMinion #:nodoc
-  VERSION = '1.2.1'
+module ParallelMinion
+  VERSION = '1.3.0'.freeze
 end

data/test/minion_scope_test.rb

@@ -3,11 +3,11 @@ require 'erb'
 require 'active_record'
 
 ActiveRecord::Base.logger         = SemanticLogger[ActiveRecord]
-ActiveRecord::Base.configurations = YAML::load(ERB.new(IO.read('test/config/database.yml')).result)
+ActiveRecord::Base.configurations = YAML.safe_load(ERB.new(IO.read('test/config/database.yml')).result)
 ActiveRecord::Base.establish_connection(:test)
 
-ActiveRecord::Schema.define :version => 0 do
-  create_table :people, :force => true do |t|
+ActiveRecord::Schema.define version: 0 do
+  create_table :people, force: true do |t|
     t.string :name
     t.string :state
     t.string :zip_code
@@ -22,12 +22,12 @@ class MinionScopeTest < Minitest::Test
     [false, true].each do |enabled|
       describe ".new with enabled: #{enabled.inspect}" do
         before do
-          Person.create(name: 'Jack', state: 'FL', zip_code: 38729)
-          Person.create(name: 'John', state: 'FL', zip_code: 35363)
-          Person.create(name: 'Jill', state: 'FL', zip_code: 73534)
-          Person.create(name: 'Joe', state: 'NY', zip_code: 45325)
-          Person.create(name: 'Jane', state: 'NY', zip_code: 45325)
-          Person.create(name: 'James', state: 'CA', zip_code: 123123)
+          Person.create(name: 'Jack', state: 'FL', zip_code: 38_729)
+          Person.create(name: 'John', state: 'FL', zip_code: 35_363)
+          Person.create(name: 'Jill', state: 'FL', zip_code: 73_534)
+          Person.create(name: 'Joe', state: 'NY', zip_code: 45_325)
+          Person.create(name: 'Jane', state: 'NY', zip_code: 45_325)
+          Person.create(name: 'James', state: 'CA', zip_code: 123_123)
           # Instruct Minions to adhere to any dynamic scopes for Person model
           ParallelMinion::Minion.scoped_classes << Person
           ParallelMinion::Minion.enabled = enabled
@@ -52,7 +52,6 @@ class MinionScopeTest < Minitest::Test
             assert_equal 3, minion.result
           end
         end
-
       end
     end
   end

data/test/minion_test.rb

@@ -6,12 +6,35 @@ class MinionTest < Minitest::Test
   include SemanticLogger::Loggable
 
   describe ParallelMinion::Minion do
+    class InMemoryAppender < SemanticLogger::Subscriber
+      attr_reader :messages
+
+      def initialize
+        @messages = []
+        self.name = 'Minion'
+      end
+
+      def log(log)
+        messages << log.dup
+      end
+    end
+
+    let :log_messages do
+      appender.messages
+    end
+
+    let :appender do
+      InMemoryAppender.new
+    end
+
+    before do
+      ParallelMinion::Minion.logger = appender
+    end
 
     [false, true].each do |enabled|
       describe enabled ? 'enabled' : 'disabled' do
         before do
           ParallelMinion::Minion.enabled = enabled
-          $log_structs.clear
         end
 
         it 'without parameters' do
@@ -33,7 +56,7 @@ class MinionTest < Minitest::Test
         end
 
         it 'raise exception' do
-          minion = ParallelMinion::Minion.new(description: 'Test') { raise "An exception" }
+          minion = ParallelMinion::Minion.new(description: 'Test') { raise 'An exception' }
           assert_raises RuntimeError do
             minion.result
           end
@@ -45,7 +68,7 @@ class MinionTest < Minitest::Test
           assert_equal name, minion.logger.name
         end
 
-        # TODO Blocks still have access to their original scope if variables cannot be
+        # TODO: Blocks still have access to their original scope if variables cannot be
         #      resolved first by the parameters, then by the values in Minion itself
         #        it 'not have access to local variables' do
         #          name = 'Jack'
@@ -73,7 +96,7 @@ class MinionTest < Minitest::Test
           SemanticLogger.tagged('TAG') do
             assert_equal 'TAG', SemanticLogger.tags.last
             minion = ParallelMinion::Minion.new(description: 'Tag Test') do
-              logger.info "Tag Test"
+              logger.info 'Tag Test'
               logger.tags.last
             end
           end
@@ -85,7 +108,7 @@ class MinionTest < Minitest::Test
           SemanticLogger.named_tagged(tag: 'TAG') do
             assert_equal({tag: 'TAG'}, SemanticLogger.named_tags)
             minion = ParallelMinion::Minion.new(description: 'Named Tags Test') do
-              logger.info "Named Tags Test"
+              logger.info 'Named Tags Test'
               SemanticLogger.named_tags
             end
           end
@@ -99,7 +122,7 @@ class MinionTest < Minitest::Test
               assert_equal({tag: 'TAG'}, SemanticLogger.named_tags)
               assert_equal 'TAG', SemanticLogger.tags.last
               minion = ParallelMinion::Minion.new(description: 'Tags Test') do
-                logger.info "Tags Test"
+                logger.info 'Tags Test'
                 [SemanticLogger.named_tags, SemanticLogger.tags.last]
               end
 
@@ -107,7 +130,6 @@ class MinionTest < Minitest::Test
               assert_equal 'TAG', minion.result.last
             end
           end
-
         end
 
         it 'include metric' do
@@ -123,23 +145,41 @@ class MinionTest < Minitest::Test
           assert_equal 456, minion.result
           assert_equal 123, hash[:value]
           assert_equal 321, value
-          SemanticLogger.flush
-          assert log = $log_structs.first, -> { $log_structs.ai }
+
+          assert log_messages.shift, -> { log_messages.ai }
+          assert completed_log = log_messages.shift, -> { log_messages.ai }
+          # Completed log message
+          assert_equal metric_name, completed_log.metric, -> { completed_log.ai }
           if enabled
-            # Completed log message
-            assert_equal metric_name, log.metric, -> { $log_structs.ai }
             # Wait log message
-            assert log = $log_structs.last, -> { $log_structs.ai }
-            assert_equal "#{metric_name}/wait", log.metric, -> { $log_structs.ai }
-          else
-            # Timeout and wait has no effect when run inline
-            assert_equal metric_name, log.metric, -> { $log_structs.ai }
+            assert waited_log = log_messages.shift, -> { log_messages.ai }
+            assert_equal "#{metric_name}/wait", waited_log.metric, -> { waited_log.ai }
           end
         end
 
+        it ':on_exception_level' do
+          minion = ParallelMinion::Minion.new(
+            description:        'Test',
+            on_exception_level: :error
+          ) do |_h|
+            raise 'Oh No'
+          end
+          # Wait for thread to complete
+          assert_raises RuntimeError do
+            minion.result
+          end
+
+          assert log_messages.shift, -> { log_messages.ai }
+          assert completed_log = log_messages.shift, -> { log_messages.ai }
+
+          assert_equal :error, completed_log.level
+          assert_equal 'Completed Test -- Exception: RuntimeError: Oh No', completed_log.message
+          refute completed_log.backtrace.empty?
+        end
+
         it 'handle multiple minions concurrently' do
           # Start 10 minions
-          minions = 10.times.collect do |i|
+          minions = Array.new(10) do |i|
             # Each Minion returns its index in the collection
             ParallelMinion::Minion.new(i, description: "Minion:#{i}") { |counter| counter }
           end
@@ -193,15 +233,13 @@ class MinionTest < Minitest::Test
         end
 
         it 'keep the original arguments' do
-          minion = ParallelMinion::Minion.new(1, 'data', 14.1, description: 'Test') do |num, str, float|
+          minion = ParallelMinion::Minion.new(1, 'data', 14.1, description: 'Test') do |num, _str, float|
             num + float
           end
           assert_equal 15.1, minion.result
           assert_equal [1, 'data', 14.1], minion.arguments
         end
       end
-
     end
-
   end
 end

data/test/test_helper.rb

@@ -1,4 +1,4 @@
-#$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+# $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
 
 require 'minitest/autorun'
 require 'parallel_minion'
@@ -6,9 +6,3 @@ require 'semantic_logger'
 
 SemanticLogger.default_level = :trace
 SemanticLogger.add_appender(file_name: 'test.log', formatter: :color)
-
-# Setup global callback for metric so that it can be tested below
-$log_structs = []
-SemanticLogger.on_metric do |log_struct|
-  $log_structs << log_struct.dup
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel_minion
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-10 00:00:00.000000000 Z
+date: 2018-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: semantic_logger
@@ -24,8 +24,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
-description: Parallel Minion allows you to take existing blocks of code and wrap them
-  in a minion so that they can run asynchronously in a separate thread
+description: 
 email:
 - reidmo@gmail.com
 executables: []
@@ -56,7 +55,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -64,13 +63,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
-summary: Pragmatic approach to parallel and asynchronous processing in Ruby
+summary: Wrap Ruby code with a minion so that it is run on a parallel thread
 test_files:
-- test/config/database.yml
-- test/minion_scope_test.rb
 - test/minion_test.rb
+- test/minion_scope_test.rb
+- test/config/database.yml
 - test/test_db.sqlite3
 - test/test_helper.rb