rocketjob 6.0.0.rc1 → 6.0.1

data/lib/rocket_job/jobs/re_encrypt/relational_job.rb

@@ -1,6 +1,3 @@
-require "active_record"
-require "sync_attr"
-
 # Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
 #
 # Run in Rails console:
@@ -11,117 +8,126 @@ require "sync_attr"
 # * This job will find any column in the database that starts with`encrypted_`.
 # * This means that temporary or other tables not part of the application tables will also be processed.
 # * Since it automatically finds and re-encrypts any column, new columns are handled without any manual intervention.
-module RocketJob
-  module Jobs
-    module ReEncrypt
-      class RelationalJob < RocketJob::Job
-        include RocketJob::Batch
-
-        self.priority                 = 30
-        self.destroy_on_complete      = false
-        self.throttle_running_jobs    = 1
-        self.throttle_running_workers = 10
-
-        input_category slice_size: 1_000
-
-        # Name of the table being re-encrypted
-        field :table_name, type: String
-
-        # Limit the number of records to re-encrypt in test environments
-        field :limit, type: Integer
-
-        validates_presence_of :table_name
-        before_batch :upload_records
-
-        # Returns [Hash] of table names with each entry being an array
-        # of columns that start with encrypted_
-        sync_cattr_reader :encrypted_columns do
-          h = {}
-          connection.tables.each do |table|
-            columns = connection.columns(table)
-            columns.each do |column|
-              if column.name.start_with?("encrypted_")
-                add_column = column.name
-                (h[table] ||= []) << add_column if add_column
+if defined?(ActiveRecord) && defined?(SyncAttr)
+  require "active_record"
+  require "sync_attr"
+
+  module RocketJob
+    module Jobs
+      module ReEncrypt
+        class RelationalJob < RocketJob::Job
+          include RocketJob::Batch
+
+          self.priority                 = 30
+          self.destroy_on_complete      = false
+          self.throttle_running_jobs    = 1
+          self.throttle_running_workers = 10
+
+          input_category slice_size: 1_000
+
+          # Name of the table being re-encrypted
+          field :table_name, type: String
+
+          # Limit the number of records to re-encrypt in test environments
+          field :limit, type: Integer
+
+          validates_presence_of :table_name
+          before_batch :upload_records
+
+          # Returns [Hash] of table names with each entry being an array
+          # of columns that start with encrypted_
+          sync_cattr_reader :encrypted_columns do
+            h = {}
+            connection.tables.each do |table|
+              columns = connection.columns(table)
+              columns.each do |column|
+                if column.name.start_with?("encrypted_")
+                  add_column = column.name
+                  (h[table] ||= []) << add_column if add_column
+                end
               end
             end
+            h
           end
-          h
-        end
 
-        # Re-encrypt all `encrypted_` columns in the relational database.
-        # Queues a Job for each table that needs re-encryption.
-        def self.start(**args)
-          encrypted_columns.keys.collect do |table|
-            create!(table_name: table, description: table, **args)
+          # Re-encrypt all `encrypted_` columns in the relational database.
+          # Queues a Job for each table that needs re-encryption.
+          def self.start(**args)
+            encrypted_columns.keys.collect do |table|
+              create!(table_name: table, description: table, **args)
+            end
           end
-        end
 
-        # Re-encrypt all encrypted columns for the named table.
-        # Does not use AR models since we do not have models for all tables.
-        def perform(range)
-          start_id, end_id = range
+          # Re-encrypt all encrypted columns for the named table.
+          # Does not use AR models since we do not have models for all tables.
+          def perform(range)
+            start_id, end_id = range
 
-          columns = self.class.encrypted_columns[table_name]
-          unless columns&.size&.positive?
-            logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
-            return
-          end
+            columns = self.class.encrypted_columns[table_name]
+            unless columns&.size&.positive?
+              logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
+              return
+            end
 
-          logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
-          sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
-
-          # Use AR to fetch all the records
-          self.class.connection.select_rows(sql).each do |row|
-            row.unshift(nil)
-            index   = 1
-            sql     = "update #{quoted_table_name} set "
-            updates = []
-            columns.collect do |column|
-              index += 1
-              value = row[index]
-              # Prevent re-encryption
-              unless value.blank?
-                new_value = re_encrypt(value)
-                updates << "#{column} = \"#{new_value}\"" if new_value != value
+            logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
+            sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
+
+            # Use AR to fetch all the records
+            self.class.connection.select_rows(sql).each do |row|
+              row.unshift(nil)
+              index   = 1
+              sql     = "update #{quoted_table_name} set "
+              updates = []
+              columns.collect do |column|
+                index += 1
+                value = row[index]
+                # Prevent re-encryption
+                unless value.blank?
+                  new_value = re_encrypt(value)
+                  updates << "#{column} = \"#{new_value}\"" if new_value != value
+                end
+              end
+              if updates.size.positive?
+                sql << updates.join(", ")
+                sql << " where id=#{row[1]}"
+                logger.trace sql
+                self.class.connection.execute sql
+              else
+                logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
               end
-            end
-            if updates.size.positive?
-              sql << updates.join(", ")
-              sql << " where id=#{row[1]}"
-              logger.trace sql
-              self.class.connection.execute sql
-            else
-              logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
             end
           end
-        end
 
-        # Returns a database connection.
-        #
-        # Override this method to support other ways of obtaining a thread specific database connection.
-        def self.connection
-          ActiveRecord::Base.connection
-        end
+          # Returns a database connection.
+          #
+          # Override this method to support other ways of obtaining a thread specific database connection.
+          def self.connection
+            ActiveRecord::Base.connection
+          end
 
-        private
+          private
 
-        def quoted_table_name
-          @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
-        end
+          def quoted_table_name
+            @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
+          end
 
-        def re_encrypt(encrypted_value)
-          return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
+          def re_encrypt(encrypted_value)
+            return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
 
-          SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
-        end
+            SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
+          end
 
-        # Upload range to re-encrypt all rows in the specified table.
-        def upload_records
-          start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
-          last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
-          self.record_count =
-            last_id.positive? ? (input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size) : 0
+          # Upload range to re-encrypt all rows in the specified table.
+          def upload_records
+            start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
+            last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
+            self.record_count =
+              if last_id.positive?
+                input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size
+              else
+                0
+              end
+          end
         end
       end
     end

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -57,6 +57,10 @@ module RocketJob
 
       def upload_file(job)
         if job.respond_to?(:upload)
+          # Return the database connection for this thread back to the connection pool
+          # in case the upload takes a long time and the database connection expires.
+          ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
+
           if original_file_name
             job.upload(upload_file_name, file_name: original_file_name)
           else
@@ -96,11 +100,10 @@ module RocketJob
       def file_exists
         # Only check for file existence when it is a local file
         return unless upload_file_name.is_a?(IOStreams::Paths::File)
-        if upload_file_name.to_s == ""
-          return errors.add(:upload_file_name, "Upload file name can't be blank.")
-        end
+        return errors.add(:upload_file_name, "Upload file name can't be blank.") if upload_file_name.to_s == ""
 
         return if upload_file_name.exist?
+
         errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
       rescue NotImplementedError
         nil

data/lib/rocket_job/lookup_collection.rb

@@ -22,8 +22,6 @@ module RocketJob
       find(id: id).first
     end
 
-    private
-
     # Internal class for uploading records in batches
     class BatchUploader
       attr_reader :record_count
@@ -46,7 +44,10 @@ module RocketJob
 
       def <<(record)
         raise(ArgumentError, "Record must be a Hash") unless record.is_a?(Hash)
-        raise(ArgumentError, "Record must include an :id key") unless record.key?(:id) || record.key?("id") || record.key?("_id")
+
+        unless record.key?(:id) || record.key?("id") || record.key?("_id")
+          raise(ArgumentError, "Record must include an :id key")
+        end
 
         @documents << record
         @record_count += 1

data/lib/rocket_job/plugins/cron.rb

@@ -14,41 +14,81 @@ module RocketJob
       extend ActiveSupport::Concern
 
       included do
-        include Restart
-
         field :cron_schedule, type: String, class_attribute: true, user_editable: true, copy_on_restart: true
 
+        # Whether to prevent another instance of this job from running with the exact _same_ cron schedule.
+        # Another job instance with a different `cron_schedule` string is permitted.
+        field :cron_singleton, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        # Whether to re-schedule the next job occurrence when this job starts, or when it is complete.
+        #
+        # `true`: Create a new scheduled instance of this job after it has started. (Default)
+        #   - Ensures that the next scheduled instance is not missed because the current instance is still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     _only_ if they were changed during an `after_start` callback.
+        #     Changes to these during other callbacks or during the `perform` will not be saved to the new scheduled
+        #     instance.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil`.
+        #
+        # `false`: Create a new scheduled instance of this job on `fail`, or `abort`.
+        #   - Prevents the next scheduled instance from running or being scheduled while the current instance is
+        #     still running.
+        #   - Any changes to fields marked with `copy_on_restart` of `true` will be saved to the new scheduled instance
+        #     at any time until after the job has failed, or is aborted.
+        #   - To prevent this job creating any new duplicate instances during subsequent processing,
+        #     its `cron_schedule` is set to `nil` after it fails or is aborted.
+        field :cron_after_start, type: Mongoid::Boolean, default: true, class_attribute: true, user_editable: true, copy_on_restart: true
+
         validates_each :cron_schedule do |record, attr, value|
           record.errors.add(attr, "Invalid cron_schedule: #{value.inspect}") if value && !Fugit::Cron.new(value)
         end
+        validate :rocket_job_cron_singleton_check
+
         before_save :rocket_job_cron_set_run_at
 
-        private
+        after_start :rocket_job_cron_on_start
+        after_abort :rocket_job_cron_end_state
+        after_complete :rocket_job_cron_end_state
+        after_fail :rocket_job_cron_end_state
+      end
 
-        # Prevent auto restart if this job does not have a cron schedule.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_new_instance
-        def rocket_job_restart_new_instance
-          return unless cron_schedule
+      def rocket_job_cron_set_run_at
+        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
 
-          super
-        end
+        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+      end
 
-        # On failure:
-        # - create a new instance scheduled to run in the future.
-        # - clear out the `cron_schedule` so this instance will not schedule another instance to run on completion.
-        # Overrides: RocketJob::Plugins::Restart#rocket_job_restart_abort
-        def rocket_job_restart_abort
-          return unless cron_schedule
+      private
 
-          rocket_job_restart_new_instance
-          update_attribute(:cron_schedule, nil)
+      def rocket_job_cron_on_start
+        return unless cron_schedule && cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      def rocket_job_cron_end_state
+        return unless cron_schedule && !cron_after_start
+
+        current_cron_schedule = cron_schedule
+        update_attribute(:cron_schedule, nil)
+        create_restart!(cron_schedule: current_cron_schedule)
+      end
+
+      # Returns [true|false] whether another instance of this job with the same cron schedule is already active
+      def rocket_job_cron_duplicate?
+        self.class.with(read: {mode: :primary}) do |conn|
+          conn.where(:state.in => %i[queued running failed paused], :id.ne => id, cron_schedule: cron_schedule).exists?
         end
       end
 
-      def rocket_job_cron_set_run_at
-        return if cron_schedule.nil? || !(cron_schedule_changed? && !run_at_changed?)
+      # Prevent creation of a new job when another is running with the same cron schedule.
+      def rocket_job_cron_singleton_check
+        return if cron_schedule.nil? || completed? || aborted? || !rocket_job_cron_duplicate?
 
-        self.run_at = Fugit::Cron.new(cron_schedule).next_time.to_utc_time
+        errors.add(:state, "Another instance of #{self.class.name} is already queued, running, failed, or paused with the same cron schedule: #{cron_schedule}")
       end
     end
   end

data/lib/rocket_job/plugins/job/persistence.rb

@@ -70,6 +70,29 @@ module RocketJob
           end
         end
 
+        # Create a new instance of this job, copying across only the `copy_on_restart` attributes.
+        # Copy across input and output categories to new scheduled job so that all of the
+        # settings are remembered between instance. Example: slice_size
+        def create_restart!(**overrides)
+          if expired?
+            logger.info("Job has expired. Not creating a new instance.")
+            return
+          end
+
+          job_attrs = self.class.rocket_job_restart_attributes.each_with_object({}) do |attr, attrs|
+            attrs[attr] = send(attr)
+          end
+          job_attrs.merge!(overrides)
+
+          job                   = self.class.new(job_attrs)
+          job.input_categories  = input_categories if respond_to?(:input_categories)
+          job.output_categories = output_categories if respond_to?(:output_categories)
+
+          job.save_with_retry!
+
+          logger.info("Created a new job instance: #{job.id}")
+        end
+
         # Set in-memory job to complete if `destroy_on_complete` and the job has been destroyed
         def reload
           return super unless destroy_on_complete
@@ -85,6 +108,19 @@ module RocketJob
             self
           end
         end
+
+        # Save with retry in case persistence takes a moment.
+        def save_with_retry!(retry_limit = 10, sleep_interval = 0.5)
+          count = 0
+          while count < retry_limit
+            return true if save
+
+            logger.info("Retrying to persist new scheduled instance: #{errors.messages.inspect}")
+            sleep(sleep_interval)
+            count += 1
+          end
+          save!
+        end
       end
     end
   end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -48,7 +48,7 @@ module RocketJob
           # Note: Throttles are executed in the order they are defined.
           def define_throttle(method_name, filter: :throttle_filter_class)
             # Duplicate to prevent modifying parent class throttles
-            definitions = rocket_job_throttles ? rocket_job_throttles.dup : ThrottleDefinitions.new
+            definitions = rocket_job_throttles ? rocket_job_throttles.deep_dup : ThrottleDefinitions.new
             definitions.add(method_name, filter)
             self.rocket_job_throttles = definitions
           end
@@ -57,7 +57,7 @@ module RocketJob
           def undefine_throttle(method_name)
             return unless rocket_job_throttles
 
-            definitions = rocket_job_throttles.dup
+            definitions = rocket_job_throttles.deep_dup
             definitions.remove(method_name)
             self.rocket_job_throttles = definitions
           end

data/lib/rocket_job/plugins/restart.rb

@@ -2,128 +2,21 @@ require "active_support/concern"
 
 module RocketJob
   module Plugins
-    # Automatically starts a new instance of this job anytime it fails, aborts, or completes.
-    #
-    # Notes:
-    # * Restartable jobs automatically abort if they fail. This prevents the failed job from being retried.
-    #   - To disable this behavior, add the following empty method:
-    #        def rocket_job_restart_abort
-    #        end
-    # * On destroy this job is destroyed without starting a new instance.
-    # * On Abort a new instance is created.
-    # * Include `RocketJob::Plugins::Singleton` to prevent multiple copies of a job from running at
-    #   the same time.
-    # * The job will not be restarted if:
-    #   - A validation fails after creating the new instance of this job.
-    #   - The job has expired.
-    # * Only the fields that have `copy_on_restart: true` will be passed onto the new instance of this job.
-    #
-    # Example:
-    #
-    # class RestartableJob < RocketJob::Job
-    #   include RocketJob::Plugins::Restart
-    #
-    #   # Retain the completed job under the completed tab in Rocket Job Web Interface.
-    #   self.destroy_on_complete = false
-    #
-    #   # Will be copied to the new job on restart.
-    #   field :limit, type: Integer, copy_on_restart: true
-    #
-    #   # Will _not_ be copied to the new job on restart.
-    #   field :list, type: Array, default: [1,2,3]
-    #
-    #   # Set run_at every time a new instance of the job is created.
-    #   after_initialize set_run_at, if: :new_record?
-    #
-    #   def perform
-    #     puts "The limit is #{limit}"
-    #     puts "The list is #{list}"
-    #     'DONE'
-    #   end
-    #
-    #   private
-    #
-    #   # Run this job in 30 minutes.
-    #   def set_run_at
-    #     self.run_at = 30.minutes.from_now
-    #   end
-    # end
-    #
-    # job = RestartableJob.create!(limit: 10, list: [4,5,6])
-    # job.reload.state
-    # # => :queued
-    #
-    # job.limit
-    # # => 10
-    #
-    # job.list
-    # # => [4,5,6]
-    #
-    # # Wait 30 minutes ...
-    #
-    # job.reload.state
-    # # => :completed
-    #
-    # # A new instance was automatically created.
-    # job2 = RestartableJob.last
-    # job2.state
-    # # => :queued
-    #
-    # job2.limit
-    # # => 10
-    #
-    # job2.list
-    # # => [1,2,3]
+    # @deprecated
     module Restart
       extend ActiveSupport::Concern
 
       included do
-        after_abort :rocket_job_restart_new_instance
-        after_complete :rocket_job_restart_new_instance
+        after_abort :create_restart!
+        after_complete :create_restart!
         after_fail :rocket_job_restart_abort
       end
 
       private
 
-      # Run again in the future, even if this run fails with an exception
-      def rocket_job_restart_new_instance
-        if expired?
-          logger.info("Job has expired. Not creating a new instance.")
-          return
-        end
-        job_attrs              =
-          rocket_job_restart_attributes.each_with_object({}) { |attr, attrs| attrs[attr] = send(attr) }
-        job                    = self.class.new(job_attrs)
-
-        # Copy across input and output categories to new scheduled job so that all of the
-        # settings are remembered between instance. Example: slice_size
-        job.input_categories  = input_categories if respond_to?(:input_categories)
-        job.output_categories = output_categories if respond_to?(:output_categories)
-
-        rocket_job_restart_save(job)
-      end
-
       def rocket_job_restart_abort
         new_record? ? abort : abort!
       end
-
-      # Allow Singleton to prevent the creation of a new job if one is already running
-      # Retry since the delete may not have persisted to disk yet.
-      def rocket_job_restart_save(job, retry_limit = 10, sleep_interval = 0.5)
-        count = 0
-        while count < retry_limit
-          if job.save
-            logger.info("Created a new job instance: #{job.id}")
-            return true
-          else
-            logger.info("Job already active, retrying after a short sleep")
-            sleep(sleep_interval)
-          end
-          count += 1
-        end
-        logger.error("New job instance not started: #{job.errors.messages.inspect}")
-        false
-      end
     end
   end
 end

data/lib/rocket_job/plugins/state_machine.rb

@@ -36,8 +36,8 @@ module RocketJob
           raise(ArgumentError, "Cannot supply both a method name and a block") if methods.size.positive? && block
           raise(ArgumentError, "Must supply either a method name or a block") unless methods.size.positive? || block
 
-          # TODO: Somehow get AASM to support options such as :if and :unless to be consistent with other callbacks
-          # For example:
+          # Limitation with AASM. It only supports guards on event transitions, not for callbacks.
+          # For example, AASM does not support callback options such as :if and :unless, yet Rails callbacks do.
           #    before_start :my_callback, unless: :encrypted?
           #    before_start :my_callback, if: :encrypted?
           event = aasm.state_machine.events[event_name]

data/lib/rocket_job/plugins/throttle_dependent_jobs.rb

@@ -0,0 +1,43 @@
+require "active_support/concern"
+module RocketJob
+  module Plugins
+    # Prevent this job from starting, or a batch slice from starting if the dependent jobs are running.
+    #
+    # Features:
+    # - Ensures dependent jobs won't run
+    # When the throttle has been exceeded all jobs of this class will be ignored until the
+    # next refresh. `RocketJob::Config::re_check_seconds` which by default is 60 seconds.
+    module ThrottleDependentJobs
+      extend ActiveSupport::Concern
+
+      included do
+        field :dependent_jobs, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
+
+        define_throttle :dependent_jobs_running?
+        define_batch_throttle :dependent_jobs_running? if respond_to?(:define_batch_throttle)
+      end
+
+      class_methods do
+        def depends_on_job(*jobs)
+          self.dependent_jobs = Array(jobs).collect(&:to_s)
+        end
+      end
+
+      private
+
+      # Checks if there are any dependent jobs are running
+      def dependent_jobs_running?
+        return false if dependent_jobs.blank?
+
+        jobs_count = RocketJob::Job.running.where(:_type.in => dependent_jobs).count
+        return false if jobs_count.zero?
+
+        logger.info(
+          message: "#{jobs_count} Dependent Jobs are running from #{dependent_jobs.join(', ')}",
+          metric:  "#{self.class.name}/dependent_jobs_throttle"
+        )
+        true
+      end
+    end
+  end
+end