rocketjob 5.4.0.beta1 → 6.0.0.rc2

data/lib/rocket_job/sliced/bzip2_output_slice.rb

@@ -33,8 +33,8 @@ module RocketJob
       def serialize_records
         return [] if @records.nil? || @records.empty?
 
-        lines = records.to_a.join("\n")
-        s = StringIO.new
+        lines = records.to_a.join("\n") + "\n"
+        s     = StringIO.new
         IOStreams::Bzip2::Writer.stream(s) { |io| io.write(lines) }
         BSON::Binary.new(s.string)
       end

data/lib/rocket_job/sliced/input.rb

@@ -139,7 +139,7 @@ module RocketJob
         document                 = all.queued.
                                    sort("_id" => 1).
                                    find_one_and_update(
-                                     {"$set" => {worker_name: worker_name, state: :running, started_at: Time.now}},
+                                     {"$set" => {worker_name: worker_name, state: "running", started_at: Time.now}},
                                      return_document: :after
                                    )
         document.collection_name = collection_name if document

data/lib/rocket_job/sliced/slice.rb

@@ -33,7 +33,7 @@ module RocketJob
       #
 
       # Current state, as set by AASM
-      field :state, type: Symbol, default: :queued
+      field :state, type: Mongoid::StringifiedSymbol, default: :queued
 
       # When processing started on this slice
       field :started_at, type: Time
@@ -139,18 +139,10 @@ module RocketJob
 
       # Returns [Hash] the slice as a Hash for storage purposes
       # Compresses / Encrypts the slice according to the job setting
-      if ::Mongoid::VERSION.to_i >= 6
-        def as_attributes
-          attrs            = super
-          attrs["records"] = serialize_records if @records
-          attrs
-        end
-      else
-        def as_document
-          attrs            = super
-          attrs["records"] = serialize_records if @records
-          attrs
-        end
+      def as_attributes
+        attrs            = super
+        attrs["records"] = serialize_records if @records
+        attrs
       end
 
       def inspect

data/lib/rocket_job/sliced/slices.rb

@@ -50,8 +50,8 @@ module RocketJob
 
       # Returns output slices in the order of their id
       # which is usually the order in which they were written.
-      def each
-        all.sort(id: 1).each { |document| yield(document) }
+      def each(&block)
+        all.sort(id: 1).each(&block)
       end
 
       # Insert a new slice into the collection
@@ -96,6 +96,17 @@ module RocketJob
         slice
       end
 
+      # Append to an existing slice if already present
+      def append(slice, input_slice)
+        existing_slice = all.where(id: input_slice.id).first
+        return insert(slice, input_slice) unless existing_slice
+
+        extra_records          = slice.is_a?(Slice) ? slice.records : slice
+        existing_slice.records = existing_slice.records + extra_records
+        existing_slice.save!
+        existing_slice
+      end
+
       alias << insert
 
       # Index for find_and_modify only if it is not already present
@@ -139,6 +150,7 @@ module RocketJob
       def last
         all.sort("_id" => -1).first
       end
+
       # rubocop:enable Style/RedundantSort
 
       # Returns [Array<Struct>] grouped exceptions by class name,

data/lib/rocket_job/sliced/writer/output.rb

@@ -1,30 +1,37 @@
 module RocketJob
   module Sliced
     module Writer
-      # Internal class for writing categorized results into output slices
-      class Output
+      class Null
         attr_reader :job, :categorized_records
-        attr_accessor :input_slice
-
-        # Collect output results and write to output collections
-        # iff job is collecting output
-        # Notes:
-        #   Nothing is saved if an exception is raised inside the block
-        def self.collect(job, input_slice = nil)
-          if job.collect_output?
-            writer = new(job, input_slice)
-            yield(writer)
-            writer.close
-          else
-            writer = NullWriter.new(job, input_slice)
-            yield(writer)
-          end
-        end
+        attr_accessor :input_slice, :append
 
-        def initialize(job, input_slice = nil)
+        def initialize(job, input_slice: nil, append: false)
           @job                 = job
           @input_slice         = input_slice
           @categorized_records = {}
+          @append              = append
+        end
+
+        def <<(_)
+          # noop
+        end
+
+        def close
+          # noop
+        end
+      end
+
+      # Internal class for writing categorized results into output slices
+      class Output < Null
+        # Collect output results and write to output collections
+        # iff job is collecting output
+        # Notes:
+        #   Partial slices are saved when an exception is raised inside the block
+        def self.collect(job, **args)
+          writer = job.output_categories.present? ? new(job, **args) : Null.new(job, **args)
+          yield(writer)
+        ensure
+          writer&.close
         end
 
         # Writes the supplied result, RocketJob::Batch::Result or RocketJob::Batch::Results
@@ -40,7 +47,8 @@ module RocketJob
         # Write categorized results to their relevant collections
         def close
           categorized_records.each_pair do |category, results|
-            job.output(category).insert(results, input_slice)
+            collection = job.output(category)
+            append ? collection.append(results, input_slice) : collection.insert(results, input_slice)
           end
         end
 
@@ -48,33 +56,13 @@ module RocketJob
 
         # Stores the categorized result from one result
         def extract_categorized_result(result)
-          category = :main
-          value    = result
+          named_category = :main
+          value          = result
           if result.is_a?(RocketJob::Batch::Result)
-            category = result.category
-            value    = result.value
-            raise(ArgumentError, "Invalid RocketJob Output Category: #{category}") if job.output_categories.exclude?(category)
+            named_category = result.category
+            value          = result.value
           end
-          (categorized_records[category] ||= []) << value unless value.nil? && !job.collect_nil_output?
-        end
-      end
-
-      class NullWriter
-        attr_reader :job, :categorized_records
-        attr_accessor :input_slice
-
-        def initialize(job, input_slice = nil)
-          @job                 = job
-          @input_slice         = input_slice
-          @categorized_records = {}
-        end
-
-        def <<(_)
-          # noop
-        end
-
-        def close
-          # noop
+          (categorized_records[named_category] ||= []) << value unless value.nil? && !job.output_category(named_category).nils
         end
       end
     end

data/lib/rocket_job/subscribers/server.rb

@@ -17,7 +17,7 @@ module RocketJob
 
           supervisor.logger.info("Stopping Pool")
           supervisor.worker_pool.stop
-          unless supervisor.worker_pool.living_count == 0
+          unless supervisor.worker_pool.living_count.zero?
             supervisor.logger.info("Giving pool #{wait_timeout} seconds to terminate")
             sleep(wait_timeout)
           end

data/lib/rocket_job/thread_worker.rb

@@ -0,0 +1,46 @@
+require "concurrent"
+module RocketJob
+  # ThreadWorker
+  #
+  # A worker runs on a single operating system thread.
+  # Is usually started under a Rocket Job server process.
+  class ThreadWorker < Worker
+    attr_reader :thread
+
+    def initialize(id:, server_name:)
+      super(id: id, server_name: server_name)
+      @shutdown = Concurrent::Event.new
+      @thread   = Thread.new { run }
+    end
+
+    def alive?
+      @thread.alive?
+    end
+
+    def backtrace
+      @thread.backtrace
+    end
+
+    def join(*args)
+      @thread.join(*args)
+    end
+
+    # Send each active worker the RocketJob::ShutdownException so that stops processing immediately.
+    def kill
+      @thread.raise(Shutdown, "Shutdown due to kill request for worker: #{name}") if @thread.alive?
+    end
+
+    def shutdown?
+      @shutdown.set?
+    end
+
+    def shutdown!
+      @shutdown.set
+    end
+
+    # Returns [true|false] whether the shutdown indicator was set
+    def wait_for_shutdown?(timeout = nil)
+      @shutdown.wait(timeout)
+    end
+  end
+end

data/lib/rocket_job/throttle_definitions.rb

@@ -1,6 +1,6 @@
 module RocketJob
   class ThrottleDefinitions
-    attr_reader :throttles
+    attr_accessor :throttles
 
     def initialize
       @throttles = []
@@ -35,5 +35,11 @@ module RocketJob
       end
       nil
     end
+
+    def deep_dup
+      new_defination           = dup
+      new_defination.throttles = throttles.map(&:dup)
+      new_defination
+    end
   end
 end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "5.4.0.beta1".freeze
+  VERSION = "6.0.0.rc2".freeze
 end

data/lib/rocket_job/worker.rb

@@ -1,5 +1,3 @@
-require "concurrent"
-require "forwardable"
 module RocketJob
   # Worker
   #
@@ -7,12 +5,9 @@ module RocketJob
   # Is usually started under a Rocket Job server process.
   class Worker
     include SemanticLogger::Loggable
-    include ActiveSupport::Callbacks
-
-    define_callbacks :running
 
     attr_accessor :id, :current_filter
-    attr_reader :thread, :name, :inline, :server_name
+    attr_reader :name, :server_name
 
     # Raised when a worker is killed so that it shutdown immediately, yet cleanly.
     #
@@ -21,59 +16,41 @@ module RocketJob
     class Shutdown < RuntimeError
     end
 
-    def self.before_running(*filters, &blk)
-      set_callback(:running, :before, *filters, &blk)
-    end
-
-    def self.after_running(*filters, &blk)
-      set_callback(:running, :after, *filters, &blk)
-    end
-
-    def self.around_running(*filters, &blk)
-      set_callback(:running, :around, *filters, &blk)
-    end
-
-    def initialize(id: 0, server_name: "inline:0", inline: false)
+    def initialize(id: 0, server_name: "inline:0")
       @id             = id
       @server_name    = server_name
-      @shutdown       = Concurrent::Event.new
       @name           = "#{server_name}:#{id}"
       @re_check_start = Time.now
       @current_filter = Config.filter || {}
-      @thread         = Thread.new { run } unless inline
-      @inline         = inline
     end
 
     def alive?
-      inline ? true : @thread.alive?
+      true
     end
 
     def backtrace
-      inline ? Thread.current.backtrace : @thread.backtrace
+      Thread.current.backtrace
     end
 
-    def join(*args)
-      @thread.join(*args) unless inline
+    def join(*_args)
+      true
     end
 
-    # Send each active worker the RocketJob::ShutdownException so that stops processing immediately.
     def kill
-      return true if inline
-
-      @thread.raise(Shutdown, "Shutdown due to kill request for worker: #{name}") if @thread.alive?
+      true
     end
 
     def shutdown?
-      @shutdown.set?
+      false
     end
 
     def shutdown!
-      @shutdown.set
+      true
     end
 
     # Returns [true|false] whether the shutdown indicator was set
-    def wait_for_shutdown?(timeout = nil)
-      @shutdown.wait(timeout)
+    def wait_for_shutdown?(_timeout = nil)
+      false
     end
 
     # Process jobs until it shuts down
@@ -146,6 +123,8 @@ module RocketJob
 
         # Should this job be throttled?
         next if job.fail_on_exception! { throttled_job?(job) }
+        # Job failed during throttle execution?
+        next if job.failed?
 
         # Start this job!
         job.fail_on_exception! { job.start!(name) }
@@ -171,27 +150,14 @@ module RocketJob
     # Applies the current filter to exclude filtered jobs.
     #
     # Returns nil if no jobs are available for processing.
-    if Mongoid::VERSION.to_f >= 7.1
-      def find_and_assign_job
-        SemanticLogger.silence(:info) do
-          scheduled = RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now)
-          working   = RocketJob::Job.queued.or(state: :running, sub_state: :processing)
-          query     = RocketJob::Job.and(working, scheduled)
-          query     = query.and(current_filter) unless current_filter.blank?
-          update    = {"$set" => {"worker_name" => name, "state" => "running"}}
-          query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
-        end
-      end
-    else
-      def find_and_assign_job
-        SemanticLogger.silence(:info) do
-          scheduled = {"$or" => [{run_at: nil}, {:run_at.lte => Time.now}]}
-          working   = {"$or" => [{state: :queued}, {state: :running, sub_state: :processing}]}
-          query     = RocketJob::Job.and(working, scheduled)
-          query     = query.where(current_filter) unless current_filter.blank?
-          update    = {"$set" => {"worker_name" => name, "state" => "running"}}
-          query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
-        end
+    def find_and_assign_job
+      SemanticLogger.silence(:info) do
+        scheduled = RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now)
+        working   = RocketJob::Job.queued.or(state: "running", sub_state: "processing")
+        query     = RocketJob::Job.and(working, scheduled)
+        query     = query.and(current_filter) unless current_filter.blank?
+        update    = {"$set" => {"worker_name" => name, "state" => "running"}}
+        query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
       end
     end
 

data/lib/rocket_job/worker_pool.rb

@@ -69,12 +69,10 @@ module RocketJob
     # Return [false] on timeout
     def join(timeout = 5)
       while (worker = workers.first)
-        if worker.join(timeout)
-          # Worker thread is dead
-          workers.shift
-        else
-          return false
-        end
+        return false unless worker.join(timeout)
+
+        # Worker thread is dead
+        workers.shift
       end
       true
     end
@@ -91,7 +89,7 @@ module RocketJob
     private
 
     def add_one
-      workers << Worker.new(id: next_worker_id, server_name: server_name)
+      workers << ThreadWorker.new(id: next_worker_id, server_name: server_name)
     rescue StandardError => e
       logger.fatal("Cannot start worker", e)
     end

data/lib/rocketjob.rb

@@ -3,6 +3,8 @@ require "semantic_logger"
 require "symmetric-encryption"
 require "mongoid"
 require "rocket_job/extensions/mongo/logging"
+require "rocket_job/extensions/iostreams/path"
+require "rocket_job/extensions/psych/yaml_tree"
 require "rocket_job/version"
 require "rocket_job/rocket_job"
 require "rocket_job/config"
@@ -13,28 +15,37 @@ require "rocket_job/extensions/mongoid/clients/options"
 require "rocket_job/extensions/mongoid/contextual/mongo"
 require "rocket_job/extensions/mongoid/factory"
 
-# Apply patches for deprecated Symbol type
-require "rocket_job/extensions/mongoid/remove_warnings"
+# Backport New StringifiedSymbol type in Mongoid v7.2
+require "rocket_job/extensions/mongoid/stringified_symbol" unless defined?(Mongoid::StringifiedSymbol)
 
 # @formatter:off
 module RocketJob
-  autoload :ActiveWorker,       "rocket_job/active_worker"
-  autoload :Batch,              "rocket_job/batch"
-  autoload :CLI,                "rocket_job/cli"
-  autoload :DirmonEntry,        "rocket_job/dirmon_entry"
-  autoload :Event,              "rocket_job/event"
-  autoload :Heartbeat,          "rocket_job/heartbeat"
-  autoload :Job,                "rocket_job/job"
-  autoload :JobException,       "rocket_job/job_exception"
-  autoload :Worker,             "rocket_job/worker"
-  autoload :Performance,        "rocket_job/performance"
-  autoload :Server,             "rocket_job/server"
-  autoload :Sliced,             "rocket_job/sliced"
-  autoload :Subscriber,         "rocket_job/subscriber"
-  autoload :Supervisor,         "rocket_job/supervisor"
-  autoload :ThrottleDefinition, "rocket_job/throttle_definition"
-  autoload :ThrottleDefinitions, "rocket_job/throttle_definitions"
-  autoload :WorkerPool, "rocket_job/worker_pool"
+  autoload :ActiveWorker,            "rocket_job/active_worker"
+  autoload :Batch,                   "rocket_job/batch"
+  autoload :CLI,                     "rocket_job/cli"
+  autoload :DirmonEntry,             "rocket_job/dirmon_entry"
+  autoload :Event,                   "rocket_job/event"
+  autoload :Heartbeat,               "rocket_job/heartbeat"
+  autoload :Job,                     "rocket_job/job"
+  autoload :JobException,            "rocket_job/job_exception"
+  autoload :LookupCollection,        "rocket_job/lookup_collection"
+  autoload :Worker,                  "rocket_job/worker"
+  autoload :Performance,             "rocket_job/performance"
+  autoload :RactorWorker,            "rocket_job/ractor_worker"
+  autoload :Server,                  "rocket_job/server"
+  autoload :Sliced,                  "rocket_job/sliced"
+  autoload :Subscriber,              "rocket_job/subscriber"
+  autoload :Supervisor,              "rocket_job/supervisor"
+  autoload :ThreadWorker,            "rocket_job/thread_worker"
+  autoload :ThrottleDefinition,      "rocket_job/throttle_definition"
+  autoload :ThrottleDefinitions,     "rocket_job/throttle_definitions"
+  autoload :WorkerPool,              "rocket_job/worker_pool"
+
+  module Category
+    autoload :Base,                  "rocket_job/category/base"
+    autoload :Input,                 "rocket_job/category/input"
+    autoload :Output,                "rocket_job/category/output"
+  end
 
   module Plugins
     module Job
@@ -49,38 +60,37 @@ module RocketJob
       autoload :Transaction,         "rocket_job/plugins/job/transaction"
       autoload :Worker,              "rocket_job/plugins/job/worker"
     end
-    autoload :Cron,             "rocket_job/plugins/cron"
-    autoload :Document,         "rocket_job/plugins/document"
-    autoload :ProcessingWindow, "rocket_job/plugins/processing_window"
-    autoload :Restart,          "rocket_job/plugins/restart"
-    autoload :Retry,            "rocket_job/plugins/retry"
-    autoload :Singleton,        "rocket_job/plugins/singleton"
-    autoload :StateMachine,     "rocket_job/plugins/state_machine"
-    autoload :Transaction,      "rocket_job/plugins/transaction"
+    autoload :Cron,                  "rocket_job/plugins/cron"
+    autoload :Document,              "rocket_job/plugins/document"
+    autoload :ProcessingWindow,      "rocket_job/plugins/processing_window"
+    autoload :Restart,               "rocket_job/plugins/restart"
+    autoload :Retry,                 "rocket_job/plugins/retry"
+    autoload :Singleton,             "rocket_job/plugins/singleton"
+    autoload :StateMachine,          "rocket_job/plugins/state_machine"
+    autoload :Transaction,           "rocket_job/plugins/transaction"
+    autoload :ThrottleDependentJobs, "rocket_job/plugins/throttle_dependent_jobs"
   end
 
   module Jobs
-    autoload :ActiveJob,               "rocket_job/jobs/active_job"
-    autoload :CopyFileJob,             "rocket_job/jobs/copy_file_job"
-    autoload :DirmonJob,               "rocket_job/jobs/dirmon_job"
-    autoload :OnDemandBatchJob,        "rocket_job/jobs/on_demand_batch_job"
-    autoload :OnDemandBatchTabularJob, "rocket_job/jobs/on_demand_batch_tabular_job"
-    autoload :OnDemandJob,             "rocket_job/jobs/on_demand_job"
-    autoload :HousekeepingJob,         "rocket_job/jobs/housekeeping_job"
-    autoload :PerformanceJob,          "rocket_job/jobs/performance_job"
-    autoload :SimpleJob,               "rocket_job/jobs/simple_job"
-    autoload :UploadFileJob,           "rocket_job/jobs/upload_file_job"
+    autoload :ActiveJob,             "rocket_job/jobs/active_job"
+    autoload :CopyFileJob,           "rocket_job/jobs/copy_file_job"
+    autoload :DirmonJob,             "rocket_job/jobs/dirmon_job"
+    autoload :OnDemandBatchJob,      "rocket_job/jobs/on_demand_batch_job"
+    autoload :OnDemandJob,           "rocket_job/jobs/on_demand_job"
+    autoload :HousekeepingJob,       "rocket_job/jobs/housekeeping_job"
+    autoload :PerformanceJob,        "rocket_job/jobs/performance_job"
+    autoload :SimpleJob,             "rocket_job/jobs/simple_job"
+    autoload :UploadFileJob,         "rocket_job/jobs/upload_file_job"
+
     module ReEncrypt
-      if defined?(ActiveRecord) && defined?(SyncAttr)
-        autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
-      end
+      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
     end
   end
 
   module Subscribers
-    autoload :Logger,           "rocket_job/subscribers/logger"
-    autoload :Server,           "rocket_job/subscribers/server"
-    autoload :Worker,           "rocket_job/subscribers/worker"
+    autoload :Logger,                "rocket_job/subscribers/logger"
+    autoload :Server,                "rocket_job/subscribers/server"
+    autoload :Worker,                "rocket_job/subscribers/worker"
   end
 end