rocketjob 5.4.0.beta2 → 6.0.0.rc3

data/lib/rocket_job/subscribers/server.rb

@@ -17,7 +17,7 @@ module RocketJob
 
           supervisor.logger.info("Stopping Pool")
           supervisor.worker_pool.stop
-          unless supervisor.worker_pool.living_count == 0
+          unless supervisor.worker_pool.living_count.zero?
             supervisor.logger.info("Giving pool #{wait_timeout} seconds to terminate")
             sleep(wait_timeout)
           end

data/lib/rocket_job/thread_worker.rb

@@ -0,0 +1,46 @@
+require "concurrent"
+module RocketJob
+  # ThreadWorker
+  #
+  # A worker runs on a single operating system thread.
+  # Is usually started under a Rocket Job server process.
+  class ThreadWorker < Worker
+    attr_reader :thread
+
+    def initialize(id:, server_name:)
+      super(id: id, server_name: server_name)
+      @shutdown = Concurrent::Event.new
+      @thread   = Thread.new { run }
+    end
+
+    def alive?
+      @thread.alive?
+    end
+
+    def backtrace
+      @thread.backtrace
+    end
+
+    def join(*args)
+      @thread.join(*args)
+    end
+
+    # Send each active worker the RocketJob::ShutdownException so that stops processing immediately.
+    def kill
+      @thread.raise(Shutdown, "Shutdown due to kill request for worker: #{name}") if @thread.alive?
+    end
+
+    def shutdown?
+      @shutdown.set?
+    end
+
+    def shutdown!
+      @shutdown.set
+    end
+
+    # Returns [true|false] whether the shutdown indicator was set
+    def wait_for_shutdown?(timeout = nil)
+      @shutdown.wait(timeout)
+    end
+  end
+end

data/lib/rocket_job/throttle_definitions.rb

@@ -1,6 +1,6 @@
 module RocketJob
   class ThrottleDefinitions
-    attr_reader :throttles
+    attr_accessor :throttles
 
     def initialize
       @throttles = []
@@ -35,5 +35,11 @@ module RocketJob
       end
       nil
     end
+
+    def deep_dup
+      new_defination           = dup
+      new_defination.throttles = throttles.map(&:dup)
+      new_defination
+    end
   end
 end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "5.4.0.beta2".freeze
+  VERSION = "6.0.0.rc3".freeze
 end

data/lib/rocket_job/worker.rb

@@ -1,5 +1,3 @@
-require "concurrent"
-require "forwardable"
 module RocketJob
   # Worker
   #
@@ -7,12 +5,9 @@ module RocketJob
   # Is usually started under a Rocket Job server process.
   class Worker
     include SemanticLogger::Loggable
-    include ActiveSupport::Callbacks
-
-    define_callbacks :running
 
     attr_accessor :id, :current_filter
-    attr_reader :thread, :name, :inline, :server_name
+    attr_reader :name, :server_name
 
     # Raised when a worker is killed so that it shutdown immediately, yet cleanly.
     #
@@ -21,59 +16,41 @@ module RocketJob
     class Shutdown < RuntimeError
     end
 
-    def self.before_running(*filters, &blk)
-      set_callback(:running, :before, *filters, &blk)
-    end
-
-    def self.after_running(*filters, &blk)
-      set_callback(:running, :after, *filters, &blk)
-    end
-
-    def self.around_running(*filters, &blk)
-      set_callback(:running, :around, *filters, &blk)
-    end
-
-    def initialize(id: 0, server_name: "inline:0", inline: false)
+    def initialize(id: 0, server_name: "inline:0")
       @id             = id
       @server_name    = server_name
-      @shutdown       = Concurrent::Event.new
       @name           = "#{server_name}:#{id}"
       @re_check_start = Time.now
       @current_filter = Config.filter || {}
-      @thread         = Thread.new { run } unless inline
-      @inline         = inline
     end
 
     def alive?
-      inline ? true : @thread.alive?
+      true
     end
 
     def backtrace
-      inline ? Thread.current.backtrace : @thread.backtrace
+      Thread.current.backtrace
     end
 
-    def join(*args)
-      @thread.join(*args) unless inline
+    def join(*_args)
+      true
     end
 
-    # Send each active worker the RocketJob::ShutdownException so that stops processing immediately.
     def kill
-      return true if inline
-
-      @thread.raise(Shutdown, "Shutdown due to kill request for worker: #{name}") if @thread.alive?
+      true
     end
 
     def shutdown?
-      @shutdown.set?
+      false
     end
 
     def shutdown!
-      @shutdown.set
+      true
     end
 
     # Returns [true|false] whether the shutdown indicator was set
-    def wait_for_shutdown?(timeout = nil)
-      @shutdown.wait(timeout)
+    def wait_for_shutdown?(_timeout = nil)
+      false
     end
 
     # Process jobs until it shuts down
@@ -146,6 +123,8 @@ module RocketJob
 
         # Should this job be throttled?
         next if job.fail_on_exception! { throttled_job?(job) }
+        # Job failed during throttle execution?
+        next if job.failed?
 
         # Start this job!
         job.fail_on_exception! { job.start!(name) }
@@ -171,27 +150,14 @@ module RocketJob
     # Applies the current filter to exclude filtered jobs.
     #
     # Returns nil if no jobs are available for processing.
-    if Mongoid::VERSION.to_f >= 7.1
-      def find_and_assign_job
-        SemanticLogger.silence(:info) do
-          scheduled = RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now)
-          working   = RocketJob::Job.queued.or(state: :running, sub_state: :processing)
-          query     = RocketJob::Job.and(working, scheduled)
-          query     = query.and(current_filter) unless current_filter.blank?
-          update    = {"$set" => {"worker_name" => name, "state" => "running"}}
-          query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
-        end
-      end
-    else
-      def find_and_assign_job
-        SemanticLogger.silence(:info) do
-          scheduled = {"$or" => [{run_at: nil}, {:run_at.lte => Time.now}]}
-          working   = {"$or" => [{state: :queued}, {state: :running, sub_state: :processing}]}
-          query     = RocketJob::Job.and(working, scheduled)
-          query     = query.where(current_filter) unless current_filter.blank?
-          update    = {"$set" => {"worker_name" => name, "state" => "running"}}
-          query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
-        end
+    def find_and_assign_job
+      SemanticLogger.silence(:info) do
+        scheduled = RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now)
+        working   = RocketJob::Job.queued.or(state: "running", sub_state: "processing")
+        query     = RocketJob::Job.and(working, scheduled)
+        query     = query.and(current_filter) unless current_filter.blank?
+        update    = {"$set" => {"worker_name" => name, "state" => "running"}}
+        query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
       end
     end
 

data/lib/rocket_job/worker_pool.rb

@@ -69,12 +69,10 @@ module RocketJob
     # Return [false] on timeout
     def join(timeout = 5)
       while (worker = workers.first)
-        if worker.join(timeout)
-          # Worker thread is dead
-          workers.shift
-        else
-          return false
-        end
+        return false unless worker.join(timeout)
+
+        # Worker thread is dead
+        workers.shift
       end
       true
     end
@@ -91,7 +89,7 @@ module RocketJob
     private
 
     def add_one
-      workers << Worker.new(id: next_worker_id, server_name: server_name)
+      workers << ThreadWorker.new(id: next_worker_id, server_name: server_name)
     rescue StandardError => e
       logger.fatal("Cannot start worker", e)
     end

data/lib/rocketjob.rb

@@ -3,6 +3,8 @@ require "semantic_logger"
 require "symmetric-encryption"
 require "mongoid"
 require "rocket_job/extensions/mongo/logging"
+require "rocket_job/extensions/iostreams/path"
+require "rocket_job/extensions/psych/yaml_tree"
 require "rocket_job/version"
 require "rocket_job/rocket_job"
 require "rocket_job/config"
@@ -13,28 +15,37 @@ require "rocket_job/extensions/mongoid/clients/options"
 require "rocket_job/extensions/mongoid/contextual/mongo"
 require "rocket_job/extensions/mongoid/factory"
 
-# Apply patches for deprecated Symbol type
-require "rocket_job/extensions/mongoid/remove_warnings"
+# Backport New StringifiedSymbol type in Mongoid v7.2
+require "rocket_job/extensions/mongoid/stringified_symbol" unless defined?(Mongoid::StringifiedSymbol)
 
 # @formatter:off
 module RocketJob
-  autoload :ActiveWorker,       "rocket_job/active_worker"
-  autoload :Batch,              "rocket_job/batch"
-  autoload :CLI,                "rocket_job/cli"
-  autoload :DirmonEntry,        "rocket_job/dirmon_entry"
-  autoload :Event,              "rocket_job/event"
-  autoload :Heartbeat,          "rocket_job/heartbeat"
-  autoload :Job,                "rocket_job/job"
-  autoload :JobException,       "rocket_job/job_exception"
-  autoload :Worker,             "rocket_job/worker"
-  autoload :Performance,        "rocket_job/performance"
-  autoload :Server,             "rocket_job/server"
-  autoload :Sliced,             "rocket_job/sliced"
-  autoload :Subscriber,         "rocket_job/subscriber"
-  autoload :Supervisor,         "rocket_job/supervisor"
-  autoload :ThrottleDefinition, "rocket_job/throttle_definition"
-  autoload :ThrottleDefinitions, "rocket_job/throttle_definitions"
-  autoload :WorkerPool, "rocket_job/worker_pool"
+  autoload :ActiveWorker,            "rocket_job/active_worker"
+  autoload :Batch,                   "rocket_job/batch"
+  autoload :CLI,                     "rocket_job/cli"
+  autoload :DirmonEntry,             "rocket_job/dirmon_entry"
+  autoload :Event,                   "rocket_job/event"
+  autoload :Heartbeat,               "rocket_job/heartbeat"
+  autoload :Job,                     "rocket_job/job"
+  autoload :JobException,            "rocket_job/job_exception"
+  autoload :LookupCollection,        "rocket_job/lookup_collection"
+  autoload :Worker,                  "rocket_job/worker"
+  autoload :Performance,             "rocket_job/performance"
+  autoload :RactorWorker,            "rocket_job/ractor_worker"
+  autoload :Server,                  "rocket_job/server"
+  autoload :Sliced,                  "rocket_job/sliced"
+  autoload :Subscriber,              "rocket_job/subscriber"
+  autoload :Supervisor,              "rocket_job/supervisor"
+  autoload :ThreadWorker,            "rocket_job/thread_worker"
+  autoload :ThrottleDefinition,      "rocket_job/throttle_definition"
+  autoload :ThrottleDefinitions,     "rocket_job/throttle_definitions"
+  autoload :WorkerPool,              "rocket_job/worker_pool"
+
+  module Category
+    autoload :Base,                  "rocket_job/category/base"
+    autoload :Input,                 "rocket_job/category/input"
+    autoload :Output,                "rocket_job/category/output"
+  end
 
   module Plugins
     module Job
@@ -49,38 +60,37 @@ module RocketJob
       autoload :Transaction,         "rocket_job/plugins/job/transaction"
       autoload :Worker,              "rocket_job/plugins/job/worker"
     end
-    autoload :Cron,             "rocket_job/plugins/cron"
-    autoload :Document,         "rocket_job/plugins/document"
-    autoload :ProcessingWindow, "rocket_job/plugins/processing_window"
-    autoload :Restart,          "rocket_job/plugins/restart"
-    autoload :Retry,            "rocket_job/plugins/retry"
-    autoload :Singleton,        "rocket_job/plugins/singleton"
-    autoload :StateMachine,     "rocket_job/plugins/state_machine"
-    autoload :Transaction,      "rocket_job/plugins/transaction"
+    autoload :Cron,                  "rocket_job/plugins/cron"
+    autoload :Document,              "rocket_job/plugins/document"
+    autoload :ProcessingWindow,      "rocket_job/plugins/processing_window"
+    autoload :Restart,               "rocket_job/plugins/restart"
+    autoload :Retry,                 "rocket_job/plugins/retry"
+    autoload :Singleton,             "rocket_job/plugins/singleton"
+    autoload :StateMachine,          "rocket_job/plugins/state_machine"
+    autoload :Transaction,           "rocket_job/plugins/transaction"
+    autoload :ThrottleDependentJobs, "rocket_job/plugins/throttle_dependent_jobs"
   end
 
   module Jobs
-    autoload :ActiveJob,               "rocket_job/jobs/active_job"
-    autoload :CopyFileJob,             "rocket_job/jobs/copy_file_job"
-    autoload :DirmonJob,               "rocket_job/jobs/dirmon_job"
-    autoload :OnDemandBatchJob,        "rocket_job/jobs/on_demand_batch_job"
-    autoload :OnDemandBatchTabularJob, "rocket_job/jobs/on_demand_batch_tabular_job"
-    autoload :OnDemandJob,             "rocket_job/jobs/on_demand_job"
-    autoload :HousekeepingJob,         "rocket_job/jobs/housekeeping_job"
-    autoload :PerformanceJob,          "rocket_job/jobs/performance_job"
-    autoload :SimpleJob,               "rocket_job/jobs/simple_job"
-    autoload :UploadFileJob,           "rocket_job/jobs/upload_file_job"
+    autoload :ActiveJob,             "rocket_job/jobs/active_job"
+    autoload :CopyFileJob,           "rocket_job/jobs/copy_file_job"
+    autoload :DirmonJob,             "rocket_job/jobs/dirmon_job"
+    autoload :OnDemandBatchJob,      "rocket_job/jobs/on_demand_batch_job"
+    autoload :OnDemandJob,           "rocket_job/jobs/on_demand_job"
+    autoload :HousekeepingJob,       "rocket_job/jobs/housekeeping_job"
+    autoload :PerformanceJob,        "rocket_job/jobs/performance_job"
+    autoload :SimpleJob,             "rocket_job/jobs/simple_job"
+    autoload :UploadFileJob,         "rocket_job/jobs/upload_file_job"
+
     module ReEncrypt
-      if defined?(ActiveRecord) && defined?(SyncAttr)
-        autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
-      end
+      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
     end
   end
 
   module Subscribers
-    autoload :Logger,           "rocket_job/subscribers/logger"
-    autoload :Server,           "rocket_job/subscribers/server"
-    autoload :Worker,           "rocket_job/subscribers/worker"
+    autoload :Logger,                "rocket_job/subscribers/logger"
+    autoload :Server,                "rocket_job/subscribers/server"
+    autoload :Worker,                "rocket_job/subscribers/worker"
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 5.4.0.beta2
+  version: 6.0.0.rc3
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-23 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.12'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.12'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -39,75 +39,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: iostreams
+  name: fugit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: mongoid
+  name: iostreams
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: semantic_logger
+  name: mongoid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
-  name: symmetric-encryption
+  name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.7'
 - !ruby/object:Gem::Dependency
-  name: fugit
+  name: symmetric-encryption
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '4.3'
 description:
 email:
 executables:
@@ -124,6 +124,7 @@ files:
 - lib/rocket_job/active_worker.rb
 - lib/rocket_job/batch.rb
 - lib/rocket_job/batch/callbacks.rb
+- lib/rocket_job/batch/categories.rb
 - lib/rocket_job/batch/io.rb
 - lib/rocket_job/batch/logger.rb
 - lib/rocket_job/batch/lower_priority.rb
@@ -140,30 +141,36 @@ files:
 - lib/rocket_job/batch/throttle_running_workers.rb
 - lib/rocket_job/batch/throttle_windows.rb
 - lib/rocket_job/batch/worker.rb
+- lib/rocket_job/category/base.rb
+- lib/rocket_job/category/input.rb
+- lib/rocket_job/category/output.rb
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
 - lib/rocket_job/dirmon_entry.rb
 - lib/rocket_job/event.rb
+- lib/rocket_job/extensions/iostreams/path.rb
 - lib/rocket_job/extensions/mongo/logging.rb
 - lib/rocket_job/extensions/mongoid/clients/options.rb
 - lib/rocket_job/extensions/mongoid/contextual/mongo.rb
 - lib/rocket_job/extensions/mongoid/factory.rb
-- lib/rocket_job/extensions/mongoid/remove_warnings.rb
+- lib/rocket_job/extensions/mongoid/stringified_symbol.rb
+- lib/rocket_job/extensions/psych/yaml_tree.rb
 - lib/rocket_job/extensions/rocket_job_adapter.rb
 - lib/rocket_job/heartbeat.rb
 - lib/rocket_job/job.rb
 - lib/rocket_job/job_exception.rb
 - lib/rocket_job/jobs/active_job.rb
+- lib/rocket_job/jobs/conversion_job.rb
 - lib/rocket_job/jobs/copy_file_job.rb
 - lib/rocket_job/jobs/dirmon_job.rb
 - lib/rocket_job/jobs/housekeeping_job.rb
 - lib/rocket_job/jobs/on_demand_batch_job.rb
-- lib/rocket_job/jobs/on_demand_batch_tabular_job.rb
 - lib/rocket_job/jobs/on_demand_job.rb
 - lib/rocket_job/jobs/performance_job.rb
 - lib/rocket_job/jobs/re_encrypt/relational_job.rb
 - lib/rocket_job/jobs/simple_job.rb
 - lib/rocket_job/jobs/upload_file_job.rb
+- lib/rocket_job/lookup_collection.rb
 - lib/rocket_job/performance.rb
 - lib/rocket_job/plugins/cron.rb
 - lib/rocket_job/plugins/document.rb
@@ -180,7 +187,9 @@ files:
 - lib/rocket_job/plugins/retry.rb
 - lib/rocket_job/plugins/singleton.rb
 - lib/rocket_job/plugins/state_machine.rb
+- lib/rocket_job/plugins/throttle_dependent_jobs.rb
 - lib/rocket_job/plugins/transaction.rb
+- lib/rocket_job/ractor_worker.rb
 - lib/rocket_job/railtie.rb
 - lib/rocket_job/rocket_job.rb
 - lib/rocket_job/server.rb
@@ -202,13 +211,14 @@ files:
 - lib/rocket_job/subscribers/worker.rb
 - lib/rocket_job/supervisor.rb
 - lib/rocket_job/supervisor/shutdown.rb
+- lib/rocket_job/thread_worker.rb
 - lib/rocket_job/throttle_definition.rb
 - lib/rocket_job/throttle_definitions.rb
 - lib/rocket_job/version.rb
 - lib/rocket_job/worker.rb
 - lib/rocket_job/worker_pool.rb
 - lib/rocketjob.rb
-homepage: http://rocketjob.io
+homepage: https://rocketjob.io
 licenses:
 - Apache-2.0
 metadata: {}
@@ -220,14 +230,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.2.15
 signing_key:
 specification_version: 4
 summary: Ruby's missing batch processing system.