rocketjob 3.0.5 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61ffcbdf9443ca63dc699d732c9cc5c228b3505a
-  data.tar.gz: cbecb1f0d5df94328fa4190847aa8422211294a4
+  metadata.gz: 90895c69510bfbacc92d44ee5145106c5ab07dac
+  data.tar.gz: 68297e81bd9d77e56c224443558c4db24c0ba701
 SHA512:
-  metadata.gz: 5e6de043e3a10b052855dfc5db392501bfad718242eecd1ef78110400e94d9fd571a88928877d613da0d520dbb984ed59cf61c66df18ce041fc85cd4028f135a
-  data.tar.gz: b1c2c436df6ceb73176a7cb65a343ab0a3c4b682099043270a8cc9409b79e272e029d2bc1b27b8e987629976030b929c3b3863bdf8a751e488c826d3ff5ec1c5
+  metadata.gz: 2bcd1137b8bdfa9fd2627c1a1d7f94bd051b9cf1868b159c5c2024932d4f12bb33d9e5151fcdf07599fc992b547717fc1b20bb1fc3a5a89b57b8d8c7b349b0b6
+  data.tar.gz: 07ebf671c8d45cf9e255cb079a1b31e2c26810fc4a94bcde1758d4b51b9e41ff8760e94e0f2a473c94d471a3723fb0e3a19ec65045ced24aa4c09c476a767263

data/lib/rocket_job/config.rb

@@ -15,9 +15,7 @@ module RocketJob
       end
     end
 
-    # Useful for Testing, not recommended elsewhere.
-    # When enabled all calls to `perform_later` will be redirected to `perform_now`.
-    # Also, exceptions will be raised instead of failing the job.
+    # DEPRECATED
     cattr_accessor(:inline_mode) { false }
 
     store_in collection: 'rocket_job.configs'

data/lib/rocket_job/job.rb

@@ -11,5 +11,6 @@ module RocketJob
     include Plugins::Job::StateMachine
     include Plugins::Job::Worker
     include Plugins::Job::Throttle
+    include Plugins::Job::ThrottleRunningJobs
   end
 end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -3,71 +3,96 @@ require 'active_support/concern'
 module RocketJob
   module Plugins
     module Job
-      # Throttle number of jobs of a specific class that are processed at the same time.
+      # Rocket Job Throttling Framework.
       #
       # Example:
+      #   # Do not run this job when the MySQL slave delay exceeds 5 minutes.
       #   class MyJob < RocketJob
-      #     # Maximum number of workers to process instances of this job at the same time.
-      #     self.throttle_running_jobs = 25
+      #     # Define a custom mysql throttle
+      #     # Prevents all jobs of this class from running on the current server.
+      #     define_throttle :mysql_throttle_exceeded?
       #
       #     def perform
       #       # ....
       #     end
-      #   end
-      #
-      # Notes:
-      # - The actual number will be around this value, it con go over slightly and
-      #   can drop depending on check interval can drop slightly below this value.
-      # - By avoid hard locks and counters performance can be maintained while still
-      #   supporting good enough throttling.
-      # - If throughput is not as important as preventing brief spikes when many
-      #   workers are running, add a double check into the perform:
-      #     class MyJob < RocketJob
-      #       self.throttle_running_jobs = 25
       #
-      #       def perform
-      #         # (Optional) Prevent a brief spike from exceeding the wax worker throttle
-      #         self.class.throttle_double_check
+      #     private
       #
-      #         # ....
-      #       end
+      #     # Returns true if the MySQL slave delay exceeds 5 minutes
+      #     def mysql_throttle_exceeded?
+      #       status        = ActiveRecord::Base.connection.select_one('show slave status')
+      #       seconds_delay = Hash(status)['Seconds_Behind_Master'].to_i
+      #       seconds_delay >= 300
       #     end
+      #   end
       module Throttle
         extend ActiveSupport::Concern
 
         included do
-          class_attribute :throttle_running_jobs
-          self.throttle_running_jobs = nil
+          class_attribute :rocket_job_throttles
+          self.rocket_job_throttles = []
         end
 
-        # Throttle to add when the throttle is exceeded
-        def throttle_filter
-          {:_type.nin => [self.class.name]}
+        module ClassMethods
+          # Add a new throttle.
+          #
+          # Parameters:
+          #   method_name: [Symbol]
+          #     Name of method to call to evaluate whether a throttle has been exceeded.
+          #     Note: Must return true or false.
+          #   filter: [Symbol|Proc]
+          #     Name of method to call to return the filter when the throttle has been exceeded.
+          #     Or, a block that will return the filter.
+          #     Default: :throttle_filter_class (Throttle all jobs of this class)
+          #
+          # Note: Throttles are executed in the order they are defined.
+          def define_throttle(method_name, filter: :throttle_filter_class)
+            raise(ArgumentError, "Filter for #{method_name} must be a Symbol or Proc") unless filter.is_a?(Symbol) || filter.is_a?(Proc)
+            raise(ArgumentError, "Cannot define #{method_name} twice, undefine previous throttle first") if has_throttle?(method_name)
+
+            self.rocket_job_throttles += [ThrottleDefinition.new(method_name, filter)]
+          end
+
+          # Undefine a previously defined throttle
+          def undefine_throttle(method_name)
+            rocket_job_throttles.delete_if { |throttle| throttle.method_name }
+          end
+
+          # Has a throttle been defined?
+          def has_throttle?(method_name)
+            rocket_job_throttles.find { |throttle| throttle.method_name == method_name }
+          end
         end
 
-        # Returns [Boolean] whether the throttle for this job has been exceeded
-        def throttle_exceeded?
-          throttle_running_jobs && (throttle_running_jobs != 0) ? (self.class.running.where(:id.ne => id).count >= throttle_running_jobs) : false
+        # Default throttle to use when the throttle is exceeded.
+        # When the throttle has been exceeded all jobs of this class will be ignored until the
+        # next refresh. `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
+        def throttle_filter_class
+          {:_type.nin => [self.class.name]}
         end
 
-        # Prevent a brief spike from exceeding the wax worker throttle
-        def throttle_double_check(check_seconds = 1)
-          while !throttle_exceeded?
-            sleep check_seconds
-          end
+        # Filter out only this instance of the job.
+        # When the throttle has been exceeded this job will be ignored by this server until the next refresh.
+        # `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
+        def throttle_filter_id
+          {:id.nin => [id]}
         end
 
-        # Merge filter(s)
-        def throttle_merge_filter(target, source)
-          source.each_pair do |k, v|
-            target[k] =
-              if previous = target[k]
-                v.is_a?(Array) ? previous + v : v
-              else
-                v
-              end
+        private
+
+        ThrottleDefinition = Struct.new(:method_name, :filter)
+
+        # Returns the matching filter, or nil if no throttles were triggered.
+        def rocket_job_evaluate_throttles
+          rocket_job_throttles.each do |throttle|
+            # Throttle exceeded?
+            if send(throttle.method_name)
+              logger.debug { "Throttle: #{throttle.method_name} has been exceeded. #{self.class.name}:#{id}" }
+              filter = throttle.filter
+              return filter.is_a?(Proc) ? filter.call(self) : send(filter)
+            end
           end
-          target
+          nil
         end
 
       end

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -0,0 +1,45 @@
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    module Job
+      # Throttle the number of jobs of a specific class that are processed at the same time.
+      #
+      # Example:
+      #   class MyJob < RocketJob
+      #     # Maximum number of jobs of this class to process at the same time.
+      #     self.throttle_running_jobs = 25
+      #
+      #     def perform
+      #       # ....
+      #     end
+      #   end
+      #
+      # Notes:
+      # - The actual number will be around this value, it can go over slightly and
+      #   can drop depending on check interval can drop slightly below this value.
+      # - By avoiding hard locks and counters performance can be maintained while still
+      #   supporting good enough quantity throttling.
+      module ThrottleRunningJobs
+        extend ActiveSupport::Concern
+
+        included do
+          # Limit number of jobs running of this class.
+          class_attribute :throttle_running_jobs
+          self.throttle_running_jobs = nil
+
+          define_throttle :throttle_running_jobs_exceeded?
+        end
+
+        private
+
+        # Returns [Boolean] whether the throttle for this job has been exceeded
+        def throttle_running_jobs_exceeded?
+          throttle_running_jobs &&
+            (throttle_running_jobs != 0) &&
+              (self.class.running.where(:id.ne => id).count >= throttle_running_jobs)
+        end
+      end
+    end
+  end
+end

data/lib/rocket_job/plugins/job/worker.rb

@@ -8,25 +8,14 @@ module RocketJob
         extend ActiveSupport::Concern
 
         module ClassMethods
-          # Run this job later
-          #
-          # Saves it to the database for processing later by workers
-          def perform_later(args, &block)
-            if RocketJob::Config.inline_mode
-              perform_now(args, &block)
-            else
-              job = new(args)
-              block.call(job) if block
-              job.save!
-              job
-            end
-          end
-
           # Run this job now.
           #
           # The job is not saved to the database since it is processed entriely in memory
           # As a result before_save and before_destroy callbacks will not be called.
           # Validations are still called however prior to calling #perform
+          #
+          # Note:
+          # - Only batch throttles are checked when perform_now is called.
           def perform_now(args, &block)
             job = new(args)
             block.call(job) if block
@@ -56,10 +45,8 @@ module RocketJob
               when job.expired?
                 job.rocket_job_fail_on_exception!(worker_name) { job.destroy }
                 logger.info "Destroyed expired job #{job.class.name}, id:#{job.id}"
-              when job.throttle_exceeded?
-                logger.debug { "Throttle exceeded with job #{job.class.name}, id:#{job.id}" }
-                # Add jobs filter to the current filter
-                job.throttle_merge_filter(filter, job.throttle_filter)
+              when new_filter = job.send(:rocket_job_evaluate_throttles)
+                rocket_job_merge_filter(filter, new_filter)
                 # Restore retrieved job so that other workers can process it later
                 job.set(worker_name: nil, state: :queued)
               else
@@ -79,6 +66,32 @@ module RocketJob
               job.requeue!(server_name) if job.may_requeue?(server_name)
             end
           end
+
+          # DEPRECATED
+          def perform_later(args, &block)
+            if RocketJob::Config.inline_mode
+              perform_now(args, &block)
+            else
+              job = new(args)
+              block.call(job) if block
+              job.save!
+              job
+            end
+          end
+
+          private
+
+          def rocket_job_merge_filter(target, source)
+            source.each_pair do |k, v|
+              target[k] =
+                if previous = target[k]
+                  v.is_a?(Array) ? previous + v : v
+                else
+                  v
+                end
+            end
+            target
+          end
         end
 
         # Runs the job now in the current thread.

data/lib/rocket_job/plugins/state_machine.rb

@@ -1,7 +1,6 @@
 require 'thread'
 require 'active_support/concern'
 require 'aasm'
-require 'rocket_job/extensions/aasm'
 
 module RocketJob
   module Plugins
@@ -28,14 +27,9 @@ module RocketJob
     #  end
     module StateMachine
       extend ActiveSupport::Concern
-      @@aasm_mutex = Mutex.new
 
       included do
         include AASM
-        # Try to make aasm lookup thread safe
-        @aasm = Concurrent::Map.new
-        # Call aasm to create default instance upfront
-        aasm
 
         # Adds a :before or :after callback to an event
         #  state_machine_add_event_callback(:start, :before, :my_method)
@@ -43,30 +37,28 @@ module RocketJob
           raise(ArgumentError, 'Cannot supply both a method name and a block') if (methods.size > 0) && block
           raise(ArgumentError, 'Must supply either a method name or a block') unless (methods.size > 0) || block
 
-          @@aasm_mutex.synchronize do
-            # TODO Somehow get AASM to support options such as :if and :unless to be consistent with other callbacks
-            # For example:
-            #    before_start :my_callback, unless: :encrypted?
-            #    before_start :my_callback, if: :encrypted?
-            if event = aasm.state_machine.events[event_name]
-              values = Array(event.options[action])
-              code   =
-                if block
-                  block
-                else
-                  # Validate methods are any of Symbol String Proc
-                  methods.each do |method|
-                    unless method.is_a?(Symbol) || method.is_a?(String)
-                      raise(ArgumentError, "#{action}_#{event_name} currently does not support any options. Only Symbol and String method names can be supplied.")
-                    end
+          # TODO Somehow get AASM to support options such as :if and :unless to be consistent with other callbacks
+          # For example:
+          #    before_start :my_callback, unless: :encrypted?
+          #    before_start :my_callback, if: :encrypted?
+          if event = aasm.state_machine.events[event_name]
+            values = Array(event.options[action])
+            code   =
+              if block
+                block
+              else
+                # Validate methods are any of Symbol String Proc
+                methods.each do |method|
+                  unless method.is_a?(Symbol) || method.is_a?(String)
+                    raise(ArgumentError, "#{action}_#{event_name} currently does not support any options. Only Symbol and String method names can be supplied.")
                   end
-                  methods
                 end
-              action == :before ? values.push(code) : values.unshift(code)
-              event.options[action] = values.flatten.uniq
-            else
-              raise(ArgumentError, "Unknown event: #{event_name.inspect}")
-            end
+                methods
+              end
+            action == :before ? values.push(code) : values.unshift(code)
+            event.options[action] = values.flatten.uniq
+          else
+            raise(ArgumentError, "Unknown event: #{event_name.inspect}")
           end
         end
 
@@ -84,21 +76,6 @@ module RocketJob
           end
         end
 
-        # Patch AASM so that save! is called instead of save
-        # So that validations are run before job.requeue! is completed
-        # Otherwise it just fails silently
-        def aasm_write_state(state, name=:default)
-          attr_name = self.class.aasm(name).attribute_name
-          old_value = read_attribute(attr_name)
-          write_attribute(attr_name, state)
-
-          begin
-            save!
-          rescue Exception => exc
-            write_attribute(attr_name, old_value)
-            raise(exc)
-          end
-        end
       end
 
     end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob #:nodoc
-  VERSION = '3.0.5'
+  VERSION = '3.1.0'
 end

data/lib/rocketjob.rb

@@ -18,14 +18,15 @@ module RocketJob
 
   module Plugins
     module Job
-      autoload :Callbacks,      'rocket_job/plugins/job/callbacks'
-      autoload :Defaults,       'rocket_job/plugins/job/defaults'
-      autoload :StateMachine,   'rocket_job/plugins/job/state_machine'
-      autoload :Logger,         'rocket_job/plugins/job/logger'
-      autoload :Model,          'rocket_job/plugins/job/model'
-      autoload :Persistence,    'rocket_job/plugins/job/persistence'
-      autoload :Throttle,       'rocket_job/plugins/job/throttle'
-      autoload :Worker,         'rocket_job/plugins/job/worker'
+      autoload :Callbacks,           'rocket_job/plugins/job/callbacks'
+      autoload :Defaults,            'rocket_job/plugins/job/defaults'
+      autoload :StateMachine,        'rocket_job/plugins/job/state_machine'
+      autoload :Logger,              'rocket_job/plugins/job/logger'
+      autoload :Model,               'rocket_job/plugins/job/model'
+      autoload :Persistence,         'rocket_job/plugins/job/persistence'
+      autoload :Throttle,            'rocket_job/plugins/job/throttle'
+      autoload :ThrottleRunningJobs, 'rocket_job/plugins/job/throttle_running_jobs'
+      autoload :Worker,              'rocket_job/plugins/job/worker'
     end
     module Rufus
       autoload :CronLine,       'rocket_job/plugins/rufus/cron_line'

data/test/plugins/job/throttle_test.rb

@@ -19,18 +19,35 @@ module Plugins
           RocketJob::Job.delete_all
         end
 
-        describe '#throttle_exceeded?' do
+        describe '.has_throttle?' do
+          it 'defines the running jobs throttle' do
+            assert ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
+            refute ThrottleJob.has_throttle?(:blah?), ThrottleJob.rocket_job_throttles
+          end
+        end
+
+        describe '.undefine_throttle' do
+          it 'undefines the running jobs throttle' do
+            assert ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
+            ThrottleJob.undefine_throttle(:throttle_running_jobs_exceeded?)
+            refute ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
+            ThrottleJob.define_throttle(:throttle_running_jobs_exceeded?)
+            assert ThrottleJob.has_throttle?(:throttle_running_jobs_exceeded?), ThrottleJob.rocket_job_throttles
+          end
+        end
+
+        describe '#throttle_running_jobs_exceeded??' do
           it 'does not exceed throttle when no other jobs are running' do
             ThrottleJob.create!
             job = ThrottleJob.new
-            refute job.throttle_exceeded?
+            refute job.send(:throttle_running_jobs_exceeded?)
           end
 
           it 'exceeds throttle when other jobs are running' do
             job1 = ThrottleJob.new
             job1.start!
             job2 = ThrottleJob.new
-            assert job2.throttle_exceeded?
+            assert job2.send(:throttle_running_jobs_exceeded?)
           end
 
           it 'excludes paused jobs' do
@@ -38,7 +55,7 @@ module Plugins
             job1.start
             job1.pause!
             job2 = ThrottleJob.new
-            refute job2.throttle_exceeded?
+            refute job2.send(:throttle_running_jobs_exceeded?)
           end
 
           it 'excludes failed jobs' do
@@ -46,7 +63,7 @@ module Plugins
             job1.start
             job1.fail!
             job2 = ThrottleJob.new
-            refute job2.throttle_exceeded?
+            refute job2.send(:throttle_running_jobs_exceeded?)
           end
         end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 3.0.5
+  version: 3.1.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-16 00:00:00.000000000 Z
+date: 2017-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '4.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '4.12'
 description: 
 email:
 - support@rocketjob.io
@@ -84,7 +84,6 @@ files:
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
 - lib/rocket_job/dirmon_entry.rb
-- lib/rocket_job/extensions/aasm.rb
 - lib/rocket_job/extensions/mongo/logging.rb
 - lib/rocket_job/extensions/rocket_job_adapter.rb
 - lib/rocket_job/heartbeat.rb
@@ -102,6 +101,7 @@ files:
 - lib/rocket_job/plugins/job/persistence.rb
 - lib/rocket_job/plugins/job/state_machine.rb
 - lib/rocket_job/plugins/job/throttle.rb
+- lib/rocket_job/plugins/job/throttle_running_jobs.rb
 - lib/rocket_job/plugins/job/worker.rb
 - lib/rocket_job/plugins/processing_window.rb
 - lib/rocket_job/plugins/restart.rb

data/lib/rocket_job/extensions/aasm.rb

@@ -1,72 +0,0 @@
-require 'aasm'
-
-# The following patches can be removed once the following PR has been merged into AASM:
-#   https://github.com/aasm/aasm/pull/269
-
-AASM::Core::Event
-module AASM::Core
-  class Event
-    def initialize_copy(orig)
-      super
-      @transitions = @transitions.collect { |transition| transition.clone }
-      @guards      = @guards.dup
-      @unless      = @unless.dup
-      @options     = {}
-      orig.options.each_pair { |name, setting| @options[name] = setting.is_a?(Hash) || setting.is_a?(Array) ? setting.dup : setting }
-    end
-  end
-end
-
-AASM::Core::State
-module AASM::Core
-  class State
-    # called internally by Ruby 1.9 after clone()
-    def initialize_copy(orig)
-      super
-      @options = {}
-      orig.options.each_pair { |name, setting| @options[name] = setting.is_a?(Hash) || setting.is_a?(Array) ? setting.dup : setting }
-    end
-  end
-end
-
-AASM::Core::Transition
-module AASM::Core
-  class Transition
-    def initialize_copy(orig)
-      super
-      @guards = @guards.dup
-      @unless = @unless.dup
-      @opts   = {}
-      orig.opts.each_pair { |name, setting| @opts[name] = setting.is_a?(Hash) || setting.is_a?(Array) ? setting.dup : setting }
-    end
-  end
-end
-
-AASM::StateMachine
-module AASM
-  class StateMachine
-    def initialize_copy(orig)
-      super
-      @states = orig.states.collect { |state| state.clone }
-      @events = {}
-      orig.events.each_pair { |name, event| @events[name] = event.clone }
-      @global_callbacks = @global_callbacks.dup
-    end
-  end
-end
-
-# Patch to try and make AASM threadsafe
-AASM::StateMachineStore
-module AASM
-  class StateMachineStore
-    @stores = Concurrent::Map.new
-
-    def self.stores
-      @stores
-    end
-
-    def initialize
-      @machines = Concurrent::Map.new
-    end
-  end
-end