rocketjob 4.3.0.beta2 → 5.0.0.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b5544569821640636a5dc8cd7953f2c0fbadcfa90ec1c7ac029fa4fc249d1269
-  data.tar.gz: 601088d6918f605b80565d2185872b161e1ccf27d923bd6483fe49ac8d5b2586
+  metadata.gz: 3574aea0102c28eece0aefa25631e57d6458e5e7fb1191e464e6e1f885d83128
+  data.tar.gz: 7909db1e628d5460c8500d269a9d3af7fe5a15b685065869a4b93a0919db1b63
 SHA512:
-  metadata.gz: 9eda8d1c26f5e808f5ef2c1157c032ed490daaf697040a8987951c5ad25f979783796567b536c897317445b413f774fddcbd82ed3898a4578e3de4ece7a91ed7
-  data.tar.gz: 21744b1a7fad3e03c49367888e4662d0d00f60c671829c54cd8c7174ead64a8b9ce0642c61ae37c063508d5df3c0a7280a78e2bb1f752ac3fcd8e57343bf4a99
+  metadata.gz: 22da70f046dea52843f8f3b7b215ab541eb1242db629cb2833a37d9d2afd0743334935824f48cf03ab50c2952f17b84d3fe80134ad3173c5487457dbea22c4ea
+  data.tar.gz: '045886026776a1273844b1184ac0e1ce09b681dbce377029869cceab914d23755eea956bda79bf2794f1d4b3b74ef79859d529bd175a5cef717cfa33902c0a32'

data/bin/rocketjob

@@ -4,13 +4,5 @@ require 'semantic_logger'
 require 'rocket_job/cli'
 
 # Start a rocketjob server instance from the command line
-begin
-  RocketJob::CLI.new(ARGV).run
-rescue Exception => exc
-  return if exc.class == SystemExit
-  # Failsafe logger that writes to STDERR
-  SemanticLogger.add_appender(io: STDERR, level: :error, formatter: :color)
-  SemanticLogger['RocketJob'].error('Rocket Job shutting down due to exception', exc)
-  SemanticLogger.flush
-  exit 1
-end
+# Allow Ruby to catch and display any startup exceptions
+RocketJob::CLI.new(ARGV).run

data/lib/rocket_job/batch/worker.rb

@@ -62,7 +62,7 @@ module RocketJob
           end
 
           # Allow new jobs with a higher priority to interrupt this job
-          break if (Time.now - start_time) >= Config.instance.re_check_seconds
+          break if (Time.now - start_time) >= Config.re_check_seconds
         end
         false
       end

data/lib/rocket_job/cli.rb

@@ -8,13 +8,10 @@ module RocketJob
   # Command Line Interface parser for Rocket Job
   class CLI
     include SemanticLogger::Loggable
-    attr_accessor :name, :workers, :environment, :pidfile, :directory, :quiet,
-                  :log_level, :log_file, :mongo_config, :symmetric_encryption_config,
-                  :include_filter, :exclude_filter, :where_filter
+    attr_accessor :environment, :pidfile, :directory, :quiet,
+                  :log_level, :log_file, :mongo_config, :symmetric_encryption_config
 
     def initialize(argv)
-      @name                        = nil
-      @workers                     = nil
       @quiet                       = false
       @environment                 = nil
       @pidfile                     = nil
@@ -40,14 +37,7 @@ module RocketJob
       # In case Rails did not load the Mongoid Config
       RocketJob::Config.load!(environment, mongo_config, symmetric_encryption_config) if ::Mongoid::Config.clients.empty?
 
-      filter = build_filter
-
-      opts               = {}
-      opts[:name]        = name if name
-      opts[:max_workers] = workers if workers
-      opts[:filter]      = filter if filter
-
-      Supervisor.run(opts)
+      Supervisor.run
     end
 
     def rails?
@@ -63,6 +53,8 @@ module RocketJob
       logger.info "Loading Rails environment: #{environment}"
       RocketJob.rails!
 
+      require 'rails'
+      require 'rocket_job/railtie'
       boot_file = Pathname.new(directory).join('config/environment.rb').expand_path
       require(boot_file.to_s)
 
@@ -153,38 +145,27 @@ module RocketJob
       end
     end
 
-    # Returns [Hash] a where clause filter to apply to this server.
-    # Returns nil if no filter should be applied
-    def build_filter
-      raise(ArgumentError, 'Cannot supply both a filter and an exclusion filter') if include_filter && exclude_filter
-
-      filter                  = where_filter
-      (filter ||= {})['_type'] = include_filter if include_filter
-      (filter ||= {})['_type'] = {'$not' => exclude_filter} if exclude_filter
-      filter
-    end
-
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
       parser        = OptionParser.new do |o|
         o.on('-n', '--name NAME', 'Unique Name of this server (Default: host_name:PID)') do |arg|
-          @name = arg
+          Config.name = arg
         end
         o.on('-w', '--workers COUNT', 'Number of workers (threads) to start') do |arg|
-          @workers = arg.to_i
+          Config.max_workers = arg.to_i
         end
-        o.on('-t', '--threads COUNT', 'DEPRECATED') do |arg|
-          warn '-t and --threads are deprecated, use -w or --workers'
-          @workers = arg.to_i
+        o.on('--include REGEXP', 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+          Config.include_filter = Regexp.new(arg, true)
         end
-        o.on('-F', '--filter REGEXP', 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
-          @include_filter = Regexp.new(arg, true)
+        o.on('-F', '--filter REGEXP', 'DEPRECATED. Use --include') do |arg|
+          warn '-F and --filter are deprecated, use --include'
+          Config.include_filter = Regexp.new(arg, true)
         end
         o.on('-E', '--exclude REGEXP', 'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
-          @exclude_filter = Regexp.new(arg, true)
+          Config.exclude_filter = Regexp.new(arg, true)
         end
         o.on('-W', '--where JSON', "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
-          @where_filter = JSON.parse(arg)
+          Config.where_filter = JSON.parse(arg)
         end
         o.on('-q', '--quiet', 'Do not write to stdout, only to logfile. Necessary when running as a daemon') do
           @quiet = true

data/lib/rocket_job/config.rb

@@ -1,52 +1,75 @@
 require 'yaml'
 module RocketJob
-  # Centralized Configuration for Rocket Jobs
+  # Rocket Job Configuration
   class Config
-    include Plugins::Document
-
-    # Returns the single instance of the Rocket Job Configuration for this site
-    # in a thread-safe way
-    def self.instance
-      @instance ||= begin
-        first || create
-      rescue StandardError
-        # In case another process has already created the first document
-        first
-      end
-    end
-
-    # DEPRECATED
-    cattr_accessor(:inline_mode) { false }
+    include SemanticLogger::Loggable
 
-    store_in collection: 'rocket_job.configs'
+    # [String] This Rocket Job Server name
+    class_attribute :server_name
+    self.server_name = "#{SemanticLogger.host}:#{$$}"
 
-    #
-    # Servers
-    #
+    # [Integer] The maximum number of workers to create on any one server
+    class_attribute :max_workers
+    self.max_workers = 10
 
-    # The maximum number of workers to create on any one server
-    field :max_workers, type: Integer, default: 10
+    # [Integer] Number of seconds between heartbeats from a Rocket Job Server process
+    class_attribute :heartbeat_seconds
+    self.heartbeat_seconds = 15.0
 
-    # Number of seconds between heartbeats from a Rocket Job Server process
-    field :heartbeat_seconds, type: Integer, default: 15
+    # [Integer] Maximum number of seconds a Worker will wait before checking for new jobs
+    class_attribute :max_poll_seconds
+    self.max_poll_seconds = 5.0
 
-    #
-    # Workers
-    #
-
-    # Maximum number of seconds a Worker will wait before checking for new jobs
-    field :max_poll_seconds, type: Integer, default: 5
-
-    # Number of seconds between checking for:
+    # [Integer] Number of seconds between checking for:
     # - Jobs with a higher priority
     # - If the current job has been paused, or aborted
     #
     # Making this interval too short results in too many checks for job status
     # changes instead of focusing on completing the active tasks
     #
+    # Notes:
+    # - Not all job types support pausing in the middle
+    #
+    # Default: 60 seconds between checks.
+    class_attribute :re_check_seconds
+    self.re_check_seconds = 60.0
+
+    # [Regexp] Limit this server to only those job classes that match this regular expression.
+    #
     # Note:
-    #   Not all job types support pausing in the middle
-    field :re_check_seconds, type: Integer, default: 60
+    # - Supply a case insensitive Regexp if required.
+    # - Only supply include_filter or exclude_filter, not both.
+    #
+    # Example:
+    #   # This server can only work on jobs that include anywhere
+    #   # in their names: `DirmonJob` or `WeeklyReportJob`
+    #   RocketJob::Config.include_filter = /DirmonJob|WeeklyReportJob/i
+    class_attribute :include_filter
+    self.include_filter = nil
+
+    # [Regexp] Prevent this server from working on any job classes that match this regular expression.
+    #
+    # Notes:
+    # - Supply a case insensitive Regexp if required.
+    # - Only supply include_filter or exclude_filter, not both.
+    #
+    # Example:
+    #   # This server can only work any job except that that include anywhere
+    #   # in their names: `DirmonJob` or `WeeklyReportJob`
+    #   RocketJob::Config.exclude_filter = /DirmonJob|WeeklyReportJob/i
+    class_attribute :exclude_filter
+    self.exclude_filter = nil
+
+    # [Hash] Limit this server instance to the supplied mongo query filter.
+    #
+    # Notes:
+    # - Can be supplied together with `include_filter` or `exclude_filter` above.
+    #
+    # Example:
+    #   # This server can only work on jobs with priorities between 1 and 25
+    #   RocketJob::Config.where_filter = { "priority" => {"$lte" => 25}}
+    class_attribute :where_filter
+    self.where_filter = nil
 
     # Configure Mongoid
     def self.load!(environment = 'development', file_name = nil, encryption_file_name = nil)
@@ -72,5 +95,19 @@ module RocketJob
       logger.debug "Reading SymmetricEncryption configuration from: #{config_file}"
       SymmetricEncryption.load!(config_file.to_s, environment)
     end
+
+    # Returns [Hash] the where clause built from the filters above:
+    #    include_filter, exclude_filter, and where_filter.
+    # Returns nil if no filter should be applied.
+    def self.filter
+      if include_filter && exclude_filter
+        raise(ArgumentError, 'Cannot supply both an include_filter and an exclude_filter')
+      end
+
+      filter                   = where_filter
+      (filter ||= {})['_type'] = include_filter if include_filter
+      (filter ||= {})['_type'] = {'$not' => exclude_filter} if exclude_filter
+      filter
+    end
   end
 end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -70,14 +70,14 @@ module RocketJob
 
         # Default throttle to use when the throttle is exceeded.
         # When the throttle has been exceeded all jobs of this class will be ignored until the
-        # next refresh. `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
+        # next refresh. `RocketJob::Config.re_check_seconds` which by default is 60 seconds.
         def throttle_filter_class
           {:_type.nin => [self.class.name]}
         end
 
         # Filter out only this instance of the job.
         # When the throttle has been exceeded this job will be ignored by this server until the next refresh.
-        # `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
+        # `RocketJob::Config.re_check_seconds` which by default is 60 seconds.
         def throttle_filter_id
           {:id.nin => [id]}
         end

data/lib/rocket_job/plugins/job/worker.rb

@@ -66,18 +66,6 @@ module RocketJob
             end
           end
 
-          # DEPRECATED
-          def perform_later(args, &block)
-            if RocketJob::Config.inline_mode
-              perform_now(args, &block)
-            else
-              job = new(args)
-              yield(job) if block
-              job.save!
-              job
-            end
-          end
-
           private
 
           def rocket_job_merge_filter(target, source)

data/lib/rocket_job/railtie.rb

@@ -0,0 +1,36 @@
+module RocketJob
+  class Railtie < Rails::Railtie
+    # Exposes Rocket Job's configuration to the Rails application configuration.
+    #
+    # @example Set up configuration in the Rails app.
+    #   module MyApplication
+    #     class Application < Rails::Application
+    #       # The maximum number of workers to create on any one server. (Default: 10)
+    #       config.rocket_job.max_workers = config.secret_config.fetch("rocket_job/max_workers", type: :integer, default: 10)
+    #
+    #       # Number of seconds between heartbeats from a Rocket Job Server process. (Default: 15)
+    #       config.rocket_job.heartbeat_seconds = config.secret_config.fetch("rocket_job/heartbeat_seconds", type: :float, default: 15.0)
+    #
+    #       # Maximum number of seconds a Worker will wait before checking for new jobs. (Default: 5)
+    #       config.rocket_job.max_poll_seconds = config.secret_config.fetch("rocket_job/max_poll_seconds", type: :float, default: 5.0)
+    #
+    #       # Number of seconds between checking for:
+    #       # - Jobs with a higher priority
+    #       # - If the current job has been paused, or aborted
+    #       #
+    #       # Making this interval too short results in too many checks for job status
+    #       # changes instead of focusing on completing the active tasks
+    #       #
+    #       # Note:
+    #       #   Not all job types support pausing in the middle
+    #       # Default: 60 seconds between checks.
+    #       config.rocket_job.re_check_seconds = config.secret_config.fetch("rocket_job/re_check_seconds", type: :float, default: 60.0)
+    #
+    #       config.rocket_job.include_filter    = config.secret_config["rocket_job/include_filter"]
+    #       config.rocket_job.exclude_filter    = config.secret_config["rocket_job/exclude_filter"]
+    #       config.rocket_job.where_filter      = config.secret_config["rocket_job/where_filter"]
+    #     end
+    #   end
+    config.rocket_job = Config
+  end
+end

data/lib/rocket_job/server/model.rb

@@ -14,18 +14,15 @@ module RocketJob
         #   Default: `host name:PID`
         # The unique name is used on re-start to re-queue any jobs that were being processed
         # at the time the server unexpectedly terminated, if any
-        field :name, type: String, default: -> { "#{SemanticLogger.host}:#{$$}" }
+        field :name, type: String, default: -> { Config.server_name }
 
         # The maximum number of workers this server should start
         #   If set, it will override the default value in RocketJob::Config
-        field :max_workers, type: Integer, default: -> { Config.instance.max_workers }
+        field :max_workers, type: Integer, default: -> { Config.max_workers }
 
         # When this server process was started
         field :started_at, type: Time
 
-        # Filter to apply to control which job classes this server can process
-        field :yaml_filter, type: String
-
         # The heartbeat information for this server
         embeds_one :heartbeat, class_name: 'RocketJob::Heartbeat'
 
@@ -82,7 +79,7 @@ module RocketJob
 
         # Scope for all zombie servers
         def self.zombies(missed = 4)
-          dead_seconds        = Config.instance.heartbeat_seconds * missed
+          dead_seconds        = Config.heartbeat_seconds * missed
           last_heartbeat_time = Time.now - dead_seconds
           where(
             :state.in => %i[stopping running paused],
@@ -94,15 +91,6 @@ module RocketJob
         end
       end
 
-      # Where clause filter to apply to workers looking for jobs
-      def filter
-        YAML.load(yaml_filter) if yaml_filter
-      end
-
-      def filter=(hash)
-        self.yaml_filter = hash.nil? ? nil : hash.to_yaml
-      end
-
       # Returns [true|false] if this server has missed at least the last 4 heartbeats
       #
       # Possible causes for a server to miss its heartbeats:
@@ -112,7 +100,7 @@ module RocketJob
       def zombie?(missed = 4)
         return false unless running? || stopping? || paused?
         return true if heartbeat.nil? || heartbeat.updated_at.nil?
-        dead_seconds = Config.instance.heartbeat_seconds * missed
+        dead_seconds = Config.heartbeat_seconds * missed
         (Time.now - heartbeat.updated_at) >= dead_seconds
       end
 

data/lib/rocket_job/supervisor.rb

@@ -10,12 +10,12 @@ module RocketJob
     attr_accessor :worker_id
 
     # Start the Supervisor, using the supplied attributes to create a new Server instance.
-    def self.run(attrs = {})
+    def self.run
       Thread.current.name = 'rocketjob main'
       RocketJob.create_indexes
       register_signal_handlers
 
-      server = Server.create!(attrs)
+      server = Server.create!
       new(server).run
     ensure
       server&.destroy
@@ -23,13 +23,13 @@ module RocketJob
 
     def initialize(server)
       @server      = server
-      @worker_pool = WorkerPool.new(server.name, server.filter)
+      @worker_pool = WorkerPool.new(server.name)
       @mutex       = Mutex.new
     end
 
     def run
       logger.info "Using MongoDB Database: #{RocketJob::Job.collection.database.name}"
-      logger.info('Running with filter', server.filter) if server.filter
+      logger.info('Running with filter', Config.filter) if Config.filter
       server.started!
       logger.info 'Rocket Job Server started'
 
@@ -81,7 +81,7 @@ module RocketJob
           end
         end
 
-        self.class.wait_for_event(Config.instance.heartbeat_seconds)
+        self.class.wait_for_event(Config.heartbeat_seconds)
 
         break if self.class.shutdown?
 

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '4.3.0.beta2'.freeze
+  VERSION = '5.0.0.beta'.freeze
 end

data/lib/rocket_job/worker.rb

@@ -11,7 +11,7 @@ module RocketJob
 
     define_callbacks :running
 
-    attr_accessor :id, :re_check_seconds, :filter, :current_filter
+    attr_accessor :id, :current_filter
     attr_reader :thread, :name, :inline
 
     # Raised when a worker is killed so that it shutdown immediately, yet cleanly.
@@ -33,21 +33,15 @@ module RocketJob
       set_callback(:running, :around, *filters, &blk)
     end
 
-    def initialize(id: 0,
-                   server_name: 'inline:0',
-                   inline: false,
-                   re_check_seconds: Config.instance.re_check_seconds,
-                   filter: nil)
-      @id               = id
-      @server_name      = server_name
-      @shutdown         = Concurrent::Event.new
-      @name             = "#{server_name}:#{id}"
-      @re_check_seconds = (re_check_seconds || 60).to_f
-      @re_check_start   = Time.now
-      @filter           = filter.nil? ? {} : filter.dup
-      @current_filter   = @filter.dup
-      @thread           = Thread.new { run } unless inline
-      @inline           = inline
+    def initialize(id: 0, server_name: 'inline:0', inline: false)
+      @id             = id
+      @server_name    = server_name
+      @shutdown       = Concurrent::Event.new
+      @name           = "#{server_name}:#{id}"
+      @re_check_start = Time.now
+      @current_filter = Config.filter || {}
+      @thread         = Thread.new { run } unless inline
+      @inline         = inline
     end
 
     def alive?
@@ -93,7 +87,7 @@ module RocketJob
       Thread.current.name = format('rocketjob %03i', id)
       logger.info 'Started'
       until shutdown?
-        wait = RocketJob::Config.instance.max_poll_seconds
+        wait = Config.max_poll_seconds
         if process_available_jobs
           # Keeps workers staggered across the poll interval so that
           # all workers don't poll at the same time
@@ -131,10 +125,10 @@ module RocketJob
     def reset_filter_if_expired
       # Only clear out the current_filter after every `re_check_seconds`
       time = Time.now
-      return unless (time - @re_check_start) > re_check_seconds
+      return unless (time - @re_check_start) > Config.re_check_seconds
 
       @re_check_start     = time
-      self.current_filter = filter.dup if current_filter != filter
+      self.current_filter = Config.filter
     end
   end
 end

data/lib/rocket_job/worker_pool.rb

@@ -5,11 +5,10 @@ module RocketJob
   class WorkerPool
     include SemanticLogger::Loggable
 
-    attr_reader :server_name, :filter, :workers
+    attr_reader :server_name, :workers
 
-    def initialize(server_name, filter)
+    def initialize(server_name)
       @server_name = server_name
-      @filter      = filter
       @workers     = Concurrent::Array.new
       @worker_id   = 0
     end
@@ -34,7 +33,7 @@ module RocketJob
 
       add_one
       count -= 1
-      delay = Config.instance.max_poll_seconds.to_f / max_workers
+      delay = Config.max_poll_seconds.to_f / max_workers
 
       count.times.each do
         sleep(delay) if stagger_start
@@ -90,7 +89,7 @@ module RocketJob
     private
 
     def add_one
-      workers << Worker.new(id: next_worker_id, server_name: server_name, filter: filter)
+      workers << Worker.new(id: next_worker_id, server_name: server_name)
     rescue StandardError => exc
       logger.fatal('Cannot start worker', exc)
     end

data/lib/rocketjob.rb

@@ -4,6 +4,8 @@ require 'mongoid'
 require 'rocket_job/extensions/mongo/logging'
 require 'rocket_job/version'
 require 'rocket_job/rocket_job'
+require 'rocket_job/config'
+require 'rocket_job/railtie' if defined?(Rails)
 
 # Apply patches to implement `with_collection`
 if Mongoid::VERSION.to_i >= 6
@@ -21,7 +23,6 @@ module RocketJob
   autoload :ActiveWorker,       'rocket_job/active_worker'
   autoload :Batch,              'rocket_job/batch'
   autoload :CLI,                'rocket_job/cli'
-  autoload :Config,             'rocket_job/config'
   autoload :DirmonEntry,        'rocket_job/dirmon_entry'
   autoload :Event,              'rocket_job/event'
   autoload :Heartbeat,          'rocket_job/heartbeat'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 4.3.0.beta2
+  version: 5.0.0.beta
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-24 00:00:00.000000000 Z
+date: 2019-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -155,6 +155,7 @@ files:
 - lib/rocket_job/plugins/singleton.rb
 - lib/rocket_job/plugins/state_machine.rb
 - lib/rocket_job/plugins/transaction.rb
+- lib/rocket_job/railtie.rb
 - lib/rocket_job/rocket_job.rb
 - lib/rocket_job/server.rb
 - lib/rocket_job/server/model.rb