rocketjob 3.0.0.beta2 → 3.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e79fbd2ab361a944bd653f0f8f614aa0cb8f951
-  data.tar.gz: 79b67d6410a552e521dd89964103fef4400c4dee
+  metadata.gz: 2893cad7faa49cc61dc42a18e1769b7e4b0684cd
+  data.tar.gz: 0484cd6616b0d1dbef6ecd67d0235ef0d3834623
 SHA512:
-  metadata.gz: 54c2021c45ace9f90b646db94edbe54863f70b9eb6397bc6f8204842422cbc22b287c3d685397a219f53dd462b388d589dd7dbb28af6b2add95894c4cd8029f5
-  data.tar.gz: c5e18971e9c434ea64901dfe377cea4ea92685ef9d1b9c445bc37a3f43830edf15ae42e4edf23bb481576fb9557f420136d1617c48603ef75a962c406ede9986
+  metadata.gz: 13e4e156847cfebb938b7700eaddda5899b79f816d690b3555d77aa6b993cc046e76c6b796404e83f3767ce482a4db50134a7037a6e7ba3ea9fd85b296834689
+  data.tar.gz: 60856f69ce93c2c0143cbf349f8c9eb393c038e44897d3f1ab16c550cec10557656628892d92ce6fbc833c5353f7b4a936a790932c219b9df7c8384cdd6f360d

data/lib/rocket_job/cli.rb

@@ -4,7 +4,9 @@ module RocketJob
   # Command Line Interface parser for RocketJob
   class CLI
     include SemanticLogger::Loggable
-    attr_accessor :name, :workers, :environment, :pidfile, :directory, :quiet, :log_level, :log_file, :mongo_config, :symmetric_encryption_config
+    attr_accessor :name, :workers, :environment, :pidfile, :directory, :quiet,
+      :log_level, :log_file, :mongo_config, :symmetric_encryption_config,
+      :filter
 
     def initialize(argv)
       @name                        = nil
@@ -17,6 +19,7 @@ module RocketJob
       @log_file                    = nil
       @mongo_config                = nil
       @symmetric_encryption_config = nil
+      @filter                      = nil
       parse(argv)
     end
 
@@ -31,6 +34,7 @@ module RocketJob
       opts               = {}
       opts[:name]        = name if name
       opts[:max_workers] = workers if workers
+      opts[:filter]      = {:_type => filter} if filter
       Server.run(opts)
     end
 
@@ -142,6 +146,9 @@ module RocketJob
           warn '-t and --threads are deprecated, use -w or --workers'
           @workers = arg.to_i
         end
+        o.on('-F', '--filter REGEXP', 'Limit this worker to only those job classes that match this regular expression (case-insensitive)') do |arg|
+          @filter = Regexp.new(arg, true)
+        end
         o.on('-q', '--quiet', 'Do not write to stdout, only to logfile. Necessary when running as a daemon') do
           @quiet = true
         end

data/lib/rocket_job/config.rb

@@ -21,6 +21,8 @@ module RocketJob
     # Also, exceptions will be raised instead of failing the job.
     cattr_accessor(:inline_mode) { false }
 
+    store_in collection: 'rocket_job.configs'
+
     #
     # Servers
     #

data/lib/rocket_job/dirmon_entry.rb

@@ -6,6 +6,8 @@ module RocketJob
     include Plugins::Document
     include Plugins::StateMachine
 
+    store_in collection: 'rocket_job.dirmon_entries'
+
     # User defined name used to identify this DirmonEntry in Mission Control
     field :name, type: String
 

data/lib/rocket_job/job.rb

@@ -11,5 +11,6 @@ module RocketJob
     include Plugins::StateMachine
     include Plugins::Job::StateMachine
     include Plugins::Job::Worker
+    include Plugins::Job::Throttle
   end
 end

data/lib/rocket_job/performance.rb

@@ -2,41 +2,49 @@ require 'csv'
 require 'yaml'
 module RocketJob
   class Performance
-    attr_accessor :count, :worker_processes, :worker_threads, :version, :ruby, :environment, :mongo_config
+    attr_accessor :count, :servers, :workers, :version, :ruby, :environment, :mongo_config
 
     def initialize
-      @version          = RocketJob::VERSION
-      @ruby             = defined?(JRuby) ? "jruby_#{JRUBY_VERSION}" : "ruby_#{RUBY_VERSION}"
-      @count            = 100_000
-      @worker_processes = 0
-      @worker_threads   = 0
-      @environment      = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-      @mongo_config     = 'config/mongo.yml'
+      @version      = RocketJob::VERSION
+      @ruby         = defined?(JRuby) ? "jruby_#{JRUBY_VERSION}" : "ruby_#{RUBY_VERSION}"
+      @count        = 100_000
+      @servers      = 0
+      @workers      = 0
+      @environment  = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      @mongo_config = 'config/mongoid.yml'
     end
 
     def run_test_case(count = self.count)
-      self.worker_processes = RocketJob::Server.count
-      raise 'Please start workers before starting the performance test' if worker_processes == 0
+      raise 'Please start servers before starting the performance test' if RocketJob::Server.where(:state.in => ['running', 'paused']).count == 0
 
-      self.worker_processes = 0
-      self.worker_threads   = 0
-      RocketJob::Server.running.each do |worker_process|
-        unless worker_process.zombie?
-          self.worker_processes += 1
-          self.worker_threads   += worker_process.heartbeat.workers
-        end
+      self.servers = 0
+      self.workers = 0
+      RocketJob::Server.running.each do |server|
+        next if server.zombie?
+        self.servers += 1
+        self.workers += server.heartbeat.workers
       end
-      puts "Running: #{worker_threads} workers, distributed across #{worker_processes} processes"
+      puts "Running: #{workers} workers, distributed across #{servers} servers"
 
       puts 'Waiting for workers to pause'
       RocketJob::Server.pause_all
       RocketJob::Jobs::SimpleJob.delete_all
-      sleep 15
+
+      # Wait for paused workers to stop
+      loop do
+        running = 0
+        RocketJob::Server.paused.each do |server|
+          running += server.heartbeat.workers unless server.zombie?
+        end
+        puts "Waiting for #{running} workers"
+        break if running == 0
+        sleep 1
+      end
 
       puts 'Enqueuing jobs'
-      first = RocketJob::Jobs::SimpleJob.create(priority: 1, destroy_on_complete: false)
+      first = RocketJob::Jobs::SimpleJob.create!(priority: 1, destroy_on_complete: false)
       (count - 2).times { |i| RocketJob::Jobs::SimpleJob.create! }
-      last = RocketJob::Jobs::SimpleJob.create(priority: 100, destroy_on_complete: false)
+      last = RocketJob::Jobs::SimpleJob.create!(priority: 100, destroy_on_complete: false)
 
       puts 'Resuming workers'
       RocketJob::Server.resume_all
@@ -51,7 +59,7 @@ module RocketJob
 
     # Export the Results hash to a CSV file
     def export_results(results)
-      CSV.open("job_results_#{ruby}_#{worker_processes}p_#{threads}t_v#{version}.csv", 'wb') do |csv|
+      CSV.open("job_results_#{ruby}_#{servers}s_#{workers}w_v#{version}.csv", 'wb') do |csv|
         csv << results.first.keys
         results.each { |result| csv << result.values }
       end

data/lib/rocket_job/plugins/job/model.rb

@@ -148,11 +148,7 @@ module RocketJob
           # Scope for queued jobs that can run now
           # I.e. Queued jobs excluding scheduled jobs
           def queued_now
-            queued.or(
-              {:run_at.exists => false},
-              {:run_at => nil},
-              {:run_at.lte => Time.now}
-            )
+            queued.or({:run_at => nil}, {:run_at.lte => Time.now})
           end
 
           # DEPRECATED

data/lib/rocket_job/plugins/job/persistence.rb

@@ -12,24 +12,34 @@ module RocketJob
           # Store all job types in this collection
           store_in collection: 'rocket_job.jobs'
 
+          after_initialize :remove_arguments
+        end
+
+        module ClassMethods
           # Retrieves the next job to work on in priority based order
           # and assigns it to this worker
           #
           # Returns nil if no jobs are available for processing
           #
           # Parameters
-          #   worker_name [String]
+          #   worker_name: [String]
           #     Name of the worker that will be processing this job
           #
-          #   skip_job_ids [Array<BSON::ObjectId>]
-          #     Job ids to exclude when looking for the next job
-          def self.rocket_job_retrieve(worker_name, skip_job_ids = nil)
-            query  = queued_now
-            update = {'$set' => {'worker_name' => worker_name, 'state' => 'running', 'started_at' => Time.now}}
-
-            query  = query.where(:id.nin => skip_job_ids) if skip_job_ids && skip_job_ids.size > 0
-
-            query.sort(priority: 1, _id: 1).find_one_and_update(update)
+          #   filter: [Hash]
+          #     Filter to apply to the query.
+          #     For example: to exclude jobs from being returned.
+          #
+          # Example:
+          #   # Skip any job ids from the job_ids_list
+          #   filter = {:id.nin => job_ids_list}
+          #   job    = RocketJob::Job.rocket_job_retrieve('host:pid:worker', filter)
+          def rocket_job_retrieve(worker_name, filter)
+            SemanticLogger.silence(:info) do
+              query  = queued_now
+              query  = query.where(filter) unless filter.blank?
+              update = {'$set' => {'worker_name' => worker_name, 'state' => 'running', 'started_at' => Time.now}}
+              query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
+            end
           end
 
           # Returns [Hash<String:Integer>] of the number of jobs in each state
@@ -59,7 +69,7 @@ module RocketJob
           #          :running => 25,
           #          :completed => 1237
           #        }
-          def self.counts_by_state
+          def counts_by_state
             counts = {}
             collection.aggregate([
               {
@@ -86,7 +96,6 @@ module RocketJob
             end
             counts
           end
-
         end
 
         # Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
@@ -104,6 +113,13 @@ module RocketJob
           end
         end
 
+        private
+
+        # Remove old style arguments that were stored as an array
+        def remove_arguments
+          attributes.delete('arguments') unless respond_to?('arguments='.to_sym)
+        end
+
       end
     end
   end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -0,0 +1,75 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+module RocketJob
+  module Plugins
+    module Job
+      # Throttle number of jobs of a specific class that are processed at the same time.
+      #
+      # Example:
+      #   class MyJob < RocketJob
+      #     # Maximum number of workers to process instances of this job at the same time.
+      #     self.throttle_max_workers = 25
+      #
+      #     def perform
+      #       # ....
+      #     end
+      #   end
+      #
+      # Notes:
+      # - The actual number will be around this value, it con go over slightly and
+      #   can drop depending on check interval can drop slightly below this value.
+      # - By avoid hard locks and counters performance can be maintained while still
+      #   supporting good enough throttling.
+      # - If throughput is not as important as preventing brief spikes when many
+      #   workers are running, add a double check into the perform:
+      #     class MyJob < RocketJob
+      #       self.throttle_max_workers = 25
+      #
+      #       def perform
+      #         # (Optional) Prevent a brief spike from exceeding the wax worker throttle
+      #         self.class.throttle_double_check
+      #
+      #         # ....
+      #       end
+      #     end
+      module Throttle
+        extend ActiveSupport::Concern
+
+        included do
+          class_attribute :throttle_max_workers
+          self.throttle_max_workers = nil
+        end
+
+        # Throttle to add when the throttle is exceeded
+        def throttle_filter
+          {:_type.nin => self.class.name}
+        end
+
+        # Returns [Boolean] whether the throttle for this job has been exceeded
+        def throttle_exceeded?
+          throttle_max_workers && (throttle_max_workers != 0) ? (self.class.running.count >= throttle_max_workers) : false
+        end
+
+        # Prevent a brief spike from exceeding the wax worker throttle
+        def throttle_double_check(check_seconds = 1)
+          while !throttle_exceeded?
+            sleep check_seconds
+          end
+        end
+
+        private
+
+        # Merge filter(s)
+        def throttle_merge_filter(target, source)
+          source.each_key do |k, v|
+            previous  = target[k]
+            target[k] = previous ? (Array(previous) << v) : v
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/rocket_job/plugins/job/worker.rb

@@ -48,8 +48,8 @@ module RocketJob
           #
           # Note:
           #   If a job is in queued state it will be started
-          def rocket_job_next_job(worker_name, skip_job_ids = nil)
-            while (job = rocket_job_retrieve(worker_name, skip_job_ids))
+          def rocket_job_next_job(worker_name, filter = {})
+            while (job = rocket_job_retrieve(worker_name, filter))
               case
               when job.running?
                 # Batch Job
@@ -57,6 +57,11 @@ module RocketJob
               when job.expired?
                 job.rocket_job_fail_on_exception!(worker_name) { job.destroy }
                 logger.info "Destroyed expired job #{job.class.name}, id:#{job.id}"
+              when job.throttle_exceeded?
+                # Add jobs filter to the current filter
+                throttle_merge_filter(filter, job.throttle_filter)
+                # Restore retrieved job so that other workers can process it later
+                job.set(worker_name: nil, state: :queued)
               else
                 job.worker_name = worker_name
                 job.rocket_job_fail_on_exception!(worker_name) do
@@ -70,7 +75,7 @@ module RocketJob
           # Requeues all jobs that were running on a server that died
           def requeue_dead_server(server_name)
             # Need to requeue paused, failed since user may have transitioned job before it finished
-            where(:state.in => [:running, :paused, :faled]).each do |job|
+            where(:state.in => [:running, :paused, :failed]).each do |job|
               job.requeue!(server_name) if job.may_requeue?(server_name)
             end
           end
@@ -136,19 +141,24 @@ module RocketJob
         # is set in the job itself.
         #
         # Thread-safe, can be called by multiple threads at the same time
-        def rocket_job_work(worker, re_raise_exceptions = false)
+        def rocket_job_work(worker, re_raise_exceptions = false, filter = nil)
           raise(ArgumentError, 'Job must be started before calling #rocket_job_work') unless running?
           rocket_job_fail_on_exception!(worker.name, re_raise_exceptions) do
-            run_callbacks :perform do
-              # Allow callbacks to fail, complete or abort the job
-              if running?
-                ret = perform
-                if collect_output?
-                  # Result must be a Hash, if not put it in a Hash
-                  self.result = ret.is_a?(Hash) ? ret : {'result' => ret}
-                end
+            if _perform_callbacks.empty?
+              @rocket_job_output = perform
+            else
+              # Allows @rocket_job_output to be modified by after/around callbacks
+              run_callbacks(:perform) do
+                # Allow callbacks to fail, complete or abort the job
+                @rocket_job_output = perform if running?
               end
             end
+
+            if collect_output?
+              # Result must be a Hash, if not put it in a Hash
+              self.result = @rocket_job_output.is_a?(Hash) ? @rocket_job_output : {'result' => @rocket_job_output}
+            end
+
             if new_record? || destroyed?
               complete if may_complete?
             else

data/lib/rocket_job/server.rb

@@ -30,6 +30,8 @@ module RocketJob
     include Plugins::StateMachine
     include SemanticLogger::Loggable
 
+    store_in collection: 'rocket_job.servers'
+
     # Unique Name of this server instance
     #   Default: `host name:PID`
     # The unique name is used on re-start to re-queue any jobs that were being processed
@@ -43,6 +45,9 @@ module RocketJob
     # When this server process was started
     field :started_at, type: Time
 
+    # Filter to apply to control which job classes this server can process
+    field :filter, type: Hash
+
     # The heartbeat information for this server
     embeds_one :heartbeat, class_name: 'RocketJob::Heartbeat'
 
@@ -188,7 +193,6 @@ module RocketJob
 
       server = create!(attrs)
       server.send(:run)
-
     ensure
       server.destroy if server
     end
@@ -225,22 +229,31 @@ module RocketJob
       logger.info "Using MongoDB Database: #{RocketJob::Job.collection.database.name}"
       build_heartbeat(updated_at: Time.now, workers: 0)
       started!
-      adjust_workers(true)
-      logger.info "RocketJob Server started with #{workers.size} workers running"
+      logger.info 'RocketJob Server started'
 
+      stagger = true
       while running? || paused?
-        sleep Config.instance.heartbeat_seconds
-
-        find_and_update(
-          'heartbeat.updated_at' => Time.now,
-          'heartbeat.workers'    => worker_count
-        )
+        SemanticLogger.silence(:info) do
+          find_and_update(
+            'heartbeat.updated_at' => Time.now,
+            'heartbeat.workers'    => worker_count
+          )
+        end
+        if paused?
+          workers.each(&:shutdown!)
+          stagger = true
+        end
 
         # In case number of threads has been modified
-        adjust_workers
+        adjust_workers(stagger)
+        stagger = false
 
         # Stop server if shutdown indicator was set
-        stop! if self.class.shutdown? && may_stop?
+        if self.class.shutdown? && may_stop?
+          stop!
+        else
+          sleep Config.instance.heartbeat_seconds
+        end
       end
 
       logger.info 'Waiting for workers to stop'
@@ -293,7 +306,7 @@ module RocketJob
     #       that not all workers poll at the same time
     #       The worker also respond faster than max_poll_seconds when a new
     #       job is added.
-    def adjust_workers(stagger_workers=false)
+    def adjust_workers(stagger_workers = false)
       count = worker_count
       # Cleanup workers that have stopped
       if count != workers.count
@@ -301,6 +314,8 @@ module RocketJob
         workers.delete_if { |t| !t.alive? }
       end
 
+      return unless running?
+
       # Need to add more workers?
       if count < max_workers
         worker_count = max_workers - count
@@ -310,7 +325,7 @@ module RocketJob
           return if shutdown?
           # Start worker
           begin
-            workers << Worker.new(id: next_worker_id, server_name: name)
+            workers << Worker.new(id: next_worker_id, server_name: name, filter: filter)
           rescue Exception => exc
             logger.fatal('Cannot start worker', exc)
           end

data/lib/rocket_job/version.rb

@@ -1,4 +1,4 @@
 # encoding: UTF-8
 module RocketJob #:nodoc
-  VERSION = '3.0.0.beta2'
+  VERSION = '3.0.0.beta3'
 end

data/lib/rocket_job/worker.rb

@@ -17,7 +17,7 @@ module RocketJob
 
     define_callbacks :running
 
-    attr_accessor :id, :worker_name, :inline
+    attr_accessor :id, :re_check_seconds, :filter, :current_filter
     attr_reader :thread, :name
 
     def self.before_running(*filters, &blk)
@@ -32,7 +32,7 @@ module RocketJob
       set_callback(:running, :around, *filters, &blk)
     end
 
-    def initialize(id: 0, server_name: 'inline', inline: false)
+    def initialize(id: 0, server_name: 'inline', inline: false, re_check_seconds: Config.instance.re_check_seconds, filter: nil)
       @id          = id
       @server_name = server_name
       if defined?(Concurrent::JavaAtomicBoolean) || defined?(Concurrent::CAtomicBoolean)
@@ -40,8 +40,12 @@ module RocketJob
       else
         @shutdown = false
       end
-      @name   = "#{server_name}:#{id}"
-      @thread = Thread.new { run } unless inline
+      @name             = "#{server_name}:#{id}"
+      @thread           = Thread.new { run } unless inline
+      @re_check_seconds = re_check_seconds
+      @re_check_start   = Time.now
+      @filter           = filter || {}
+      @current_filter   = @filter.dup
     end
 
     if defined?(Concurrent::JavaAtomicBoolean) || defined?(Concurrent::CAtomicBoolean)
@@ -87,21 +91,23 @@ module RocketJob
     rescue Exception => exc
       logger.fatal('Unhandled exception in job processing thread', exc)
     ensure
-      # TODO: Move to after_running callback
       ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
     end
 
     # Process the next available job
     # Returns [Boolean] whether any job was actually processed
     def process_available_jobs
-      skip_job_ids = []
-      processed    = false
-      while (job = Job.rocket_job_next_job(worker_name, skip_job_ids)) && !shutdown?
+      # Only clear out the current_filter after every `re_check_seconds`
+      time = Time.now
+      if (time - @re_check_start) > re_check_seconds
+        @recheck_start      = time
+        self.current_filter = filter.dup
+      end
+
+      processed = false
+      while (job = Job.rocket_job_next_job(name, current_filter)) && !shutdown?
         logger.fast_tag("job:#{job.id}") do
-          if job.rocket_job_work(self)
-            # Need to skip the specified job due to throttling or no work available
-            skip_job_ids << job.id
-          else
+          unless job.rocket_job_work(self, false, current_filter)
             processed = true
           end
         end

data/lib/rocketjob.rb

@@ -25,6 +25,7 @@ module RocketJob
       autoload :Logger,         'rocket_job/plugins/job/logger'
       autoload :Model,          'rocket_job/plugins/job/model'
       autoload :Persistence,    'rocket_job/plugins/job/persistence'
+      autoload :Throttle,       'rocket_job/plugins/job/throttle'
       autoload :Worker,         'rocket_job/plugins/job/worker'
     end
     module Rufus

data/test/plugins/job/throttle_test.rb

@@ -0,0 +1,56 @@
+require_relative '../../test_helper'
+
+# Unit Test for RocketJob::Job
+module Plugins
+  module Job
+    class ThrottleTest < Minitest::Test
+
+      class ThrottleJob < RocketJob::Job
+        # Only allow one to be processed at a time
+        self.throttle_max_workers = 1
+
+        def perform
+          21
+        end
+      end
+
+      describe RocketJob::Plugins::Job::Logger do
+        before do
+          ThrottleJob.delete_all
+        end
+
+        describe '#throttle_exceeded?' do
+          it 'does not exceed throttle when no other jobs are running' do
+            ThrottleJob.create!
+            job = ThrottleJob.new
+            refute job.throttle_exceeded?
+          end
+
+          it 'exceeds throttle when other jobs are running' do
+            job1 = ThrottleJob.new
+            job1.start!
+            job2 = ThrottleJob.new
+            assert job2.throttle_exceeded?
+          end
+
+          it 'excludes paused jobs' do
+            job1 = ThrottleJob.new
+            job1.start
+            job1.pause!
+            job2 = ThrottleJob.new
+            refute job2.throttle_exceeded?
+          end
+
+          it 'excludes failed jobs' do
+            job1 = ThrottleJob.new
+            job1.start
+            job1.fail!
+            job2 = ThrottleJob.new
+            refute job2.throttle_exceeded?
+          end
+        end
+
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 3.0.0.beta2
+  version: 3.0.0.beta3
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-14 00:00:00.000000000 Z
+date: 2016-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -101,6 +101,7 @@ files:
 - lib/rocket_job/plugins/job/model.rb
 - lib/rocket_job/plugins/job/persistence.rb
 - lib/rocket_job/plugins/job/state_machine.rb
+- lib/rocket_job/plugins/job/throttle.rb
 - lib/rocket_job/plugins/job/worker.rb
 - lib/rocket_job/plugins/processing_window.rb
 - lib/rocket_job/plugins/restart.rb
@@ -127,6 +128,7 @@ files:
 - test/plugins/job/model_test.rb
 - test/plugins/job/persistence_test.rb
 - test/plugins/job/state_machine_test.rb
+- test/plugins/job/throttle_test.rb
 - test/plugins/job/worker_test.rb
 - test/plugins/processing_window_test.rb
 - test/plugins/restart_test.rb
@@ -173,6 +175,7 @@ test_files:
 - test/plugins/job/model_test.rb
 - test/plugins/job/persistence_test.rb
 - test/plugins/job/state_machine_test.rb
+- test/plugins/job/throttle_test.rb
 - test/plugins/job/worker_test.rb
 - test/plugins/processing_window_test.rb
 - test/plugins/restart_test.rb