rocketjob 5.2.0.beta1 → 5.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8920ff4a319e838729863bbf2a7a66a4ee003ac9328cc1c3d3942fc59cecab2c
-  data.tar.gz: 3c5f4927370915b3188295cd181f5b1dc4ae64a17b0cd070e946cfa82b7517c6
+  metadata.gz: 42be7df83c1d38b5ef3a41751e3cc6afd6fb885638a7d144bfd1f71ff5de441e
+  data.tar.gz: fd3942b7fe1aa2d76bda3395561bea56122b9eab58c0ae65d99fbea76e3d771c
 SHA512:
-  metadata.gz: 1395412d271a4a1cde3d1f80fa6d45d5a0e948454b34e6a1d1221631b718d1f4fb16e3683a4a57af1e1f64d724219b70ab65cd84f7a2c9658d67e0608d263973
-  data.tar.gz: f5b161cf5dcb32126dff42b645b3356707ea74623f011af9a6e7bf0b8fee6403254b302ec5644a85181c207dbe70cfa3a82349be4a2bf52127e68e9f4ebaf5ab
+  metadata.gz: 324641130fd1bb0724058d81cdf33dad02bbdeef2047413a479dcb0bb0782be37be3e0aad9b5c57d9ea200e6cc30889ce06ce8130716f91399472bf317ed4089
+  data.tar.gz: dfe9e7f121e7fee9c4713d5fc819e3889d1e0d1d208ac14cd4678f83ce0f802f51e2067cd10d687301a9f7470a609eab6259036e191ae8e6198f7a7830cbbd82

data/README.md

@@ -3,9 +3,9 @@
 
 Ruby's missing batch system
 
-Checkout http://rocketjob.io/
+Checkout https://rocketjob.io/
 
-![Rocket Job](http://rocketjob.io/images/rocket/rocket-icon-512x512.png)
+![Rocket Job](https://rocketjob.io/images/rocket/rocket-icon-512x512.png)
 
 ## Documentation
 

data/lib/rocket_job/batch.rb

@@ -24,6 +24,7 @@ module RocketJob
     autoload :LowerPriority, "rocket_job/batch/lower_priority"
     autoload :Performance, "rocket_job/batch/performance"
     autoload :Statistics, "rocket_job/batch/statistics"
+    autoload :ThrottleWindows, "rocket_job/batch/throttle_windows"
     autoload :Result, "rocket_job/batch/result"
     autoload :Results, "rocket_job/batch/results"
     autoload :Tabular, "rocket_job/batch/tabular"

data/lib/rocket_job/batch/io.rb

@@ -18,7 +18,7 @@ module RocketJob
           raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}"
         end
 
-        (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(rocket_job_io_slice_arguments("inputs", category))
+        (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(**rocket_job_io_slice_arguments("inputs", category))
       end
 
       # Returns [RocketJob::Sliced::Output] output collection for holding output slices
@@ -34,7 +34,7 @@ module RocketJob
           raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}"
         end
 
-        (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(rocket_job_io_slice_arguments("outputs", category))
+        (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(**rocket_job_io_slice_arguments("outputs", category))
       end
 
       # Upload the supplied file, io, IOStreams::Path, or IOStreams::Stream.

data/lib/rocket_job/batch/throttle.rb

@@ -6,7 +6,7 @@ module RocketJob
     #
     # Example:
     #   # Do not run any slices for this job when the MySQL slave delay exceeds 5 minutes.
-    #   class MyJob < RocketJob
+    #   class MyJob < RocketJob::Job
     #     include RocketJob::Batch
     #
     #     # Define a custom mysql throttle

data/lib/rocket_job/batch/throttle_running_workers.rb

@@ -5,7 +5,7 @@ module RocketJob
     # Throttle the number of slices of a specific batch job that are processed at the same time.
     #
     # Example:
-    #   class MyJob < RocketJob
+    #   class MyJob < RocketJob::Job
     #     include RocketJob::Batch
     #
     #     # Maximum number of slices to process at the same time for each running instance.
@@ -53,6 +53,19 @@ module RocketJob
           conn.where(:id.ne => slice.id).count >= throttle_running_workers
         end
       end
+
+      # Returns [Boolean] whether the throttle for this job has been exceeded
+      #
+      # With a Batch job, allow a higher priority queued job to replace a running one with
+      # a lower priority.
+      def throttle_running_jobs_exceeded?
+        return unless throttle_running_jobs&.positive?
+
+        # Cannot use this class since it will include instances of parent job classes.
+        RocketJob::Job.with(read: {mode: :primary}) do |conn|
+          conn.running.where("_type" => self.class.name, :id.ne => id, :priority.lte => priority).count >= throttle_running_jobs
+        end
+      end
     end
   end
 end

data/lib/rocket_job/batch/throttle_windows.rb

@@ -0,0 +1,72 @@
+require "active_support/concern"
+require "fugit"
+
+module RocketJob
+  module Batch
+    # For a batch job that can run over a long period of time it can be useful
+    # to prevent its slices from being processed outside a predefined processing window.
+    #
+    # This plugin supports up to 2 different processing windows.
+    #
+    # For example, do not run this job during business hours.
+    # Allow it to run from 5pm until 8am the following day Mon through Fri.
+    #
+    #   class AfterHoursJob < RocketJob::Job
+    #     include RocketJob::Batch
+    #     include RocketJob::Batch::ThrottleWindows
+    #
+    #     # Monday through Thursday the job can start processing at 5pm Eastern.
+    #     self.primary_schedule = "0 17 * * 1-4 America/New_York"
+    #     # Slices are allowed to run until 8am the following day, which is 15 hours long:
+    #     self.primary_duration = 15.hours
+    #
+    #     # The slices for this job can run all weekend long, starting Friday at 5pm Eastern.
+    #     self.secondary_schedule = "0 17 * * 5 America/New_York"
+    #     # Slices are allowed to run until 8am on Monday morning, which is 63 hours long:
+    #     self.secondary_duration = 63.hours
+    #   end
+    #
+    # Notes:
+    # * These schedules do not affect when the job is started, completed, or when `before_batch` or
+    #   `after_batch` processing is performed. It only limits when individual slices are processed.
+    module ThrottleWindows
+      extend ActiveSupport::Concern
+
+      included do
+        # Beginning of the primary schedule. In cron format, see Scheduled Jobs `cron_schedule` for examples.
+        field :primary_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
+        # Duration in seconds of the primary window.
+        field :primary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        # Beginning of the secondary schedule. In cron format, see Scheduled Jobs `cron_schedule` for examples.
+        field :secondary_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
+        # Duration in seconds of the secondary window.
+        field :secondary_duration, type: Integer, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        define_batch_throttle :throttle_windows_exceeded?, filter: :throttle_filter_id
+
+        validates_each :primary_schedule, :secondary_schedule do |record, attr, value|
+          record.errors.add(attr, "Invalid #{attr}: #{value.inspect}") if value && !Fugit::Cron.new(value)
+        end
+      end
+
+      private
+
+      def throttle_windows_exceeded?
+        exceeded = primary_schedule && primary_duration && throttle_outside_window?(primary_schedule, primary_duration)
+        if exceeded && secondary_schedule && secondary_duration
+          exceeded = throttle_outside_window?(secondary_schedule, secondary_duration)
+        end
+        exceeded
+      end
+
+      def throttle_outside_window?(schedule, duration)
+        cron = Fugit::Cron.new(schedule)
+        time = Time.now.utc + 1
+        # Add 1 second since right now could be the very beginning of the processing window.
+        previous_time = cron.previous_time(time).to_utc_time
+        previous_time + duration < time
+      end
+    end
+  end
+end

data/lib/rocket_job/batch/worker.rb

@@ -28,10 +28,6 @@ module RocketJob
       #
       # If an exception was thrown the entire slice of records is marked as failed.
       #
-      # If the mongo_ha gem has been loaded, then the connection to mongo is
-      # automatically re-established and the job will resume anytime a
-      # Mongo connection failure occurs.
-      #
       # Thread-safe, can be called by multiple threads at the same time
       def rocket_job_work(worker, re_raise_exceptions = false)
         raise "Job must be started before calling #rocket_job_work" unless running?
@@ -114,8 +110,6 @@ module RocketJob
         servers
       end
 
-      private
-
       def rocket_job_batch_throttled?(slice, worker)
         filter = self.class.rocket_job_batch_throttles.matching_filter(self, slice)
         return false unless filter
@@ -129,46 +123,59 @@ module RocketJob
       # Process a single slice from Mongo
       # Once the slice has been successfully processed it will be removed from the input collection
       # Returns [Integer] the number of records successfully processed
-      def rocket_job_process_slice(slice)
-        # TODO: Skip records already processed
-        @rocket_job_record_number = slice.first_record_number || 0
-        @rocket_job_slice         = slice
+      def rocket_job_process_slice(slice, &block)
+        @rocket_job_slice = slice
+        count             = 0
 
-        processed_records = 0
         run_callbacks(:slice) do
           # Allow before_slice callbacks to fail, complete or abort this slice.
           return 0 unless running?
 
-          RocketJob::Sliced::Writer::Output.collect(self, slice) do |writer|
-            slice.each do |record|
-              SemanticLogger.named_tagged(record: @rocket_job_record_number) do
-                writer << rocket_job_batch_perform(slice, record)
-                processed_records += 1
-              end
-              # JRuby thinks self.rocket_job_record_number= is private and cannot be accessed
-              @rocket_job_record_number += 1
-            end
-          end
-          @rocket_job_slice         = nil
-          @rocket_job_record_number = nil
+          count = rocket_job_perform_slice(slice, &block)
         end
+        @rocket_job_slice = nil
 
         # On successful completion remove the slice from the input queue
         # TODO: Add option to complete slice instead of destroying it to retain input data.
         slice.destroy
-        processed_records
+        count
+      end
+
+      # Perform individual slice without callbacks
+      def rocket_job_perform_slice(slice, &block)
+        count = 0
+        RocketJob::Sliced::Writer::Output.collect(self, slice) do |writer|
+          records = slice.records
+
+          # Skip records already processed, if any.
+          # slice.processing_record_number ||= 0
+          # TODO: Must append to existing output slices before this can be enabled.
+          # if !collect_output && (slice.processing_record_number > 1)
+          #   records = records[slice.processing_record_number - 1..-1]
+          # end
+          # Until the changes above have been implemented, reprocess all records in the slice.
+          slice.processing_record_number = 0
+
+          records.each do |record|
+            slice.processing_record_number += 1
+            SemanticLogger.named_tagged(record: slice.current_record_number) do
+              writer << rocket_job_batch_perform(slice, record, &block)
+              count += 1
+            end
+          end
+        end
+        count
       end
 
       # Perform a single record within the current slice.
       def rocket_job_batch_perform(slice, record)
-        slice.processing_record_number ||= 0
-        slice.processing_record_number += 1
+        @rocket_job_record_number = slice.current_record_number
 
         return block_given? ? yield(record) : perform(record) if _perform_callbacks.empty?
 
         # @rocket_job_input and @rocket_job_output can be modified by before/around callbacks
-        @rocket_job_input  = record
-        @rocket_job_output = nil
+        @rocket_job_input         = record
+        @rocket_job_output        = nil
 
         run_callbacks(:perform) do
           @rocket_job_output =
@@ -179,9 +186,9 @@ module RocketJob
             end
         end
 
-        @rocket_job_input  = nil
-        result             = @rocket_job_output
-        @rocket_job_output = nil
+        @rocket_job_input         = nil
+        result                    = @rocket_job_output
+        @rocket_job_output        = nil
         result
       end
 
@@ -197,24 +204,7 @@ module RocketJob
         if failed_count.positive? && (input_count == failed_count)
           # Reload to pull in any counters or other data that was modified.
           reload unless new_record?
-          if may_fail?
-            fail_job = true
-            unless new_record?
-              # Fail job iff no other worker has already finished it
-              # Must set write concern to at least 1 since we need the nModified back
-              result   = self.class.with(write: {w: 1}) do |query|
-                query.
-                  where(id: id, state: :running, sub_state: :processing).
-                  update({"$set" => {state: :failed, worker_name: worker_name}})
-              end
-              fail_job = false unless result.modified_count.positive?
-            end
-            if fail_job
-              message        = "#{failed_count} slices failed to process"
-              self.exception = JobException.new(message: message)
-              fail!(worker_name, message)
-            end
-          end
+          rocket_job_batch_fail!(worker_name) if may_fail?
           return true
         end
 
@@ -237,15 +227,38 @@ module RocketJob
 
         # Reload to pull in any counters or other data that was modified.
         reload
+
         if result.modified_count.positive?
           rocket_job_batch_run_after_callbacks(false)
-        else
+        elsif aborted?
           # Repeat cleanup in case this worker was still running when the job was aborted
-          cleanup! if aborted?
+          cleanup!
         end
         true
       end
 
+      # Fail the job
+      def rocket_job_batch_fail!(worker_name)
+        fail_job = true
+
+        unless new_record?
+          # Fail job iff no other worker has already finished it
+          # Must set write concern to at least 1 since we need the nModified back
+          result = self.class.with(write: {w: 1}) do |query|
+            query.
+              where(id: id, state: :running, sub_state: :processing).
+              update({"$set" => {state: :failed, worker_name: worker_name}})
+          end
+          fail_job = false unless result.modified_count.positive?
+        end
+
+        return unless fail_job
+
+        message        = "#{input.failed.count} slices failed to process"
+        self.exception = JobException.new(message: message)
+        new_record? ? fail(worker_name, message) : fail!(worker_name, message)
+      end
+
       # Run the before_batch callbacks
       # Saves the current state before and after running callbacks if callbacks present
       def rocket_job_batch_run_before_callbacks

data/lib/rocket_job/event.rb

@@ -104,8 +104,6 @@ module RocketJob
       end
     end
 
-    private
-
     @load_time   = Time.now.utc
     @subscribers = Concurrent::Map.new { Concurrent::Array.new }
 

data/lib/rocket_job/extensions/mongoid/clients/options.rb

@@ -22,8 +22,6 @@ module RocketJob
         @collection_name = collection_name&.to_sym
       end
 
-      private
-
       module ClassMethods
         def with_collection(collection_name)
           all.with_collection(collection_name)

data/lib/rocket_job/plugins/cron.rb

@@ -1,4 +1,5 @@
 require "active_support/concern"
+require "fugit"
 
 module RocketJob
   module Plugins
@@ -17,7 +18,9 @@ module RocketJob
 
         field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
 
-        validate :rocket_job_cron_valid
+        validates_each :cron_schedule do |record, attr, value|
+          record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
+        end
         before_save :rocket_job_cron_set_run_at
 
         private
@@ -49,24 +52,14 @@ module RocketJob
       #     The next time as of this time.
       #     Default: Time.now
       def rocket_job_cron_next_time(time = Time.now)
-        RocketJob::Plugins::Rufus::CronLine.new(cron_schedule).next_time(time)
+        Fugit::Cron.new(cron_schedule).next_time.to_utc_time
       end
 
-      private
-
       def rocket_job_cron_set_run_at
         return unless cron_schedule
 
         self.run_at = rocket_job_cron_next_time if cron_schedule_changed? && !run_at_changed?
       end
-
-      def rocket_job_cron_valid
-        return unless cron_schedule
-
-        RocketJob::Plugins::Rufus::CronLine.new(cron_schedule)
-      rescue ArgumentError => e
-        errors.add(:cron_schedule, e.message)
-      end
     end
   end
 end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -7,7 +7,7 @@ module RocketJob
       #
       # Example:
       #   # Do not run this job when the MySQL slave delay exceeds 5 minutes.
-      #   class MyJob < RocketJob
+      #   class MyJob < RocketJob::Job
       #     # Define a custom mysql throttle
       #     # Prevents all jobs of this class from running on the current server.
       #     define_throttle :mysql_throttle_exceeded?

data/lib/rocket_job/plugins/job/throttle_running_jobs.rb

@@ -6,7 +6,7 @@ module RocketJob
       # Throttle the number of jobs of a specific class that are processed at the same time.
       #
       # Example:
-      #   class MyJob < RocketJob
+      #   class MyJob < RocketJob::Job
       #     # Maximum number of jobs of this class to process at the same time.
       #     self.throttle_running_jobs = 25
       #

data/lib/rocket_job/plugins/processing_window.rb

@@ -1,4 +1,5 @@
 require "active_support/concern"
+require "fugit"
 
 module RocketJob
   module Plugins
@@ -47,18 +48,14 @@ module RocketJob
 
         validates_presence_of :processing_schedule, :processing_duration
         validates_each :processing_schedule do |record, attr, value|
-          begin
-            RocketJob::Plugins::Rufus::CronLine.new(value)
-          rescue ArgumentError => e
-            record.errors.add(attr, e.message)
-          end
+          record.errors.add(attr, "Invalid schedule: #{value.inspect}") unless Fugit::Cron.new(value)
         end
       end
 
       # Returns [true|false] whether this job is currently inside its processing window
       def rocket_job_processing_window_active?
-        time          = Time.now
-        previous_time = rocket_job_processing_schedule.previous_time(time)
+        time          = Time.now.utc
+        previous_time = Fugit::Cron.new(processing_schedule).previous_time(time).to_utc_time
         # Inside previous processing window?
         previous_time + processing_duration > time
       end
@@ -69,17 +66,14 @@ module RocketJob
       def rocket_job_processing_window_check
         return if rocket_job_processing_window_active?
 
-        logger.warn("Processing window closed before job was processed. Job is re-scheduled to run at: #{rocket_job_processing_schedule.next_time}")
+        next_time = Fugit::Cron.new(processing_schedule).next_time.to_utc_time
+        logger.warn("Processing window closed before job was processed. Job is re-scheduled to run at: #{next_time}")
         self.worker_name ||= "inline"
         requeue!(worker_name)
       end
 
       def rocket_job_processing_window_set_run_at
-        self.run_at = rocket_job_processing_schedule.next_time unless rocket_job_processing_window_active?
-      end
-
-      def rocket_job_processing_schedule
-        RocketJob::Plugins::Rufus::CronLine.new(processing_schedule)
+        self.run_at = Fugit::Cron.new(processing_schedule).next_time.to_utc_time unless rocket_job_processing_window_active?
       end
     end
   end