rocketjob 5.4.1 → 6.0.0

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -27,15 +27,14 @@ module RocketJob
     #   )
     class HousekeepingJob < RocketJob::Job
       include RocketJob::Plugins::Cron
-      include RocketJob::Plugins::Singleton
 
-      self.priority    = 25
-      self.description = "Cleans out historical jobs, and zombie servers."
-      # Runs every 15 minutes
-      self.cron_schedule = "*/15 * * * * UTC"
+      # Runs every 15 minutes on the 15 minute period
+      self.cron_schedule = "0,15,30,45 * * * * UTC"
+      self.description   = "Cleans out historical jobs, and zombie servers."
+      self.priority      = 25
 
       # Whether to destroy zombie servers automatically
-      field :destroy_zombies, type: Boolean, default: true, user_editable: true, copy_on_restart: true
+      field :destroy_zombies, type: Mongoid::Boolean, default: true, user_editable: true, copy_on_restart: true
 
       # Retention intervals in seconds.
       # Set to nil to retain everything.
@@ -54,12 +53,12 @@ module RocketJob
         RocketJob::Job.paused.where(completed_at: {"$lte" => paused_retention.seconds.ago}).destroy_all if paused_retention
         RocketJob::Job.queued.where(created_at: {"$lte" => queued_retention.seconds.ago}).destroy_all if queued_retention
 
-        if destroy_zombies
-          # Cleanup zombie servers
-          RocketJob::Server.destroy_zombies
-          # Requeue jobs where the worker is in the zombie state and its server has gone away
-          RocketJob::ActiveWorker.requeue_zombies
-        end
+        return unless destroy_zombies
+
+        # Cleanup zombie servers
+        RocketJob::Server.destroy_zombies
+        # Requeue jobs where the worker is in the zombie state and its server has gone away
+        RocketJob::ActiveWorker.requeue_zombies
       end
     end
   end

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -31,16 +31,17 @@
 #   job.perform_now
 #   job.cleanup!
 #
-# By default output is not collected, add the option `collect_output: true` to collect output.
+# By default output is not collected, call the method `#collect_output` to collect output.
 #
 # Example:
 #   job = RocketJob::Jobs::OnDemandBatchJob(
 #     description:              'Fix data',
 #     code:                     code,
 #     throttle_running_workers: 5,
-#     priority:                 30,
-#     collect_output:           true
+#     priority:                 30
 #   )
+#   job.collect_output
+#   job.save!
 #
 # Example: Move the upload operation into a before_batch.
 #   upload_code = <<-CODE
@@ -64,27 +65,29 @@ module RocketJob
   module Jobs
     class OnDemandBatchJob < RocketJob::Job
       include RocketJob::Plugins::Cron
+      include RocketJob::Plugins::Retry
       include RocketJob::Batch
       include RocketJob::Batch::Statistics
 
       self.priority            = 90
-      self.description         = "Batch Job"
+      self.description         = "On Demand Batch Job"
       self.destroy_on_complete = false
+      self.retry_limit         = 0
 
       # Code that is performed against every row / record.
-      field :code, type: String
+      field :code, type: String, user_editable: true, copy_on_restart: true
 
       # Optional code to execute before the batch is run.
       # Usually to upload data into the job.
-      field :before_code, type: String
+      field :before_code, type: String, user_editable: true, copy_on_restart: true
 
       # Optional code to execute after the batch is run.
       # Usually to upload data into the job.
-      field :after_code, type: String
+      field :after_code, type: String, user_editable: true, copy_on_restart: true
 
       # Data that is made available to the job during the perform.
       # Be sure to store key names only as Strings, not Symbols.
-      field :data, type: Hash, default: {}
+      field :data, type: Hash, default: {}, user_editable: true, copy_on_restart: true
 
       validates :code, presence: true
       validate :validate_code
@@ -95,10 +98,20 @@ module RocketJob
       before_batch :run_before_code
       after_batch :run_after_code
 
+      # Shortcut for setting the slice_size
+      def slice_size=(slice_size)
+        input_category.slice_size = slice_size
+      end
+
+      # Add a new output category and collect output for it.
+      def add_output_category(**args)
+        self.output_categories << RocketJob::Category::Output.new(**args)
+      end
+
       private
 
       def load_perform_code
-        instance_eval("def perform(row)\n#{code}\nend")
+        instance_eval("def perform(row)\n#{code}\nend", __FILE__, __LINE__)
       end
 
       def run_before_code
@@ -118,13 +131,13 @@ module RocketJob
       def validate_before_code
         return if before_code.nil?
 
-        validate_field(:before_code) { instance_eval("def __before_code\n#{before_code}\nend") }
+        validate_field(:before_code) { instance_eval("def __before_code\n#{before_code}\nend", __FILE__, __LINE__) }
       end
 
       def validate_after_code
         return if after_code.nil?
 
-        validate_field(:after_code) { instance_eval("def __after_code\n#{after_code}\nend") }
+        validate_field(:after_code) { instance_eval("def __after_code\n#{after_code}\nend", __FILE__, __LINE__) }
       end
 
       def validate_field(field)

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -38,12 +38,11 @@
 #
 # Example: Retain output:
 #   code = <<~CODE
-#     {'value' => data['a'] * data['b']}
+#     data['result'] = data['a'] * data['b']
 #   CODE
 #
 #   RocketJob::Jobs::OnDemandJob.create!(
 #     code:           code,
-#     collect_output: true,
 #     data:           {'a' => 10, 'b' => 2}
 #   )
 #
@@ -79,8 +78,8 @@ module RocketJob
       self.retry_limit         = 0
 
       # Be sure to store key names only as Strings, not Symbols
-      field :data, type: Hash, default: {}, copy_on_restart: true
-      field :code, type: String, copy_on_restart: true
+      field :data, type: Hash, default: {}, user_editable: true, copy_on_restart: true
+      field :code, type: String, user_editable: true, copy_on_restart: true
 
       validates :code, presence: true
       validate :validate_code

data/lib/rocket_job/jobs/performance_job.rb

@@ -6,9 +6,11 @@ module RocketJob
       # Define the job's default attributes
       self.description         = "Performance Test"
       self.priority            = 5
-      self.slice_size          = 100
       self.destroy_on_complete = false
 
+      input_category slice_size: 100
+      output_category
+
       # No operation, just return the supplied line (record)
       def perform(line)
         line

data/lib/rocket_job/jobs/re_encrypt/relational_job.rb

@@ -1,6 +1,3 @@
-require "active_record"
-require "sync_attr"
-
 # Batch Worker to Re-encrypt all encrypted fields in MySQL that start with `encrytped_`.
 #
 # Run in Rails console:
@@ -11,116 +8,126 @@ require "sync_attr"
 # * This job will find any column in the database that starts with`encrypted_`.
 # * This means that temporary or other tables not part of the application tables will also be processed.
 # * Since it automatically finds and re-encrypts any column, new columns are handled without any manual intervention.
-module RocketJob
-  module Jobs
-    module ReEncrypt
-      class RelationalJob < RocketJob::Job
-        include RocketJob::Batch
-
-        self.slice_size               = 1000
-        self.priority                 = 30
-        self.destroy_on_complete      = false
-        self.compress                 = true
-        self.throttle_running_jobs    = 1
-        self.throttle_running_workers = 10
-
-        # Name of the table being re-encrypted
-        field :table_name, type: String
-
-        # Limit the number of records to re-encrypt in test environments
-        field :limit, type: Integer
-
-        validates_presence_of :table_name
-        before_batch :upload_records
-
-        # Returns [Hash] of table names with each entry being an array
-        # of columns that start with encrypted_
-        sync_cattr_reader :encrypted_columns do
-          h = {}
-          connection.tables.each do |table|
-            columns = connection.columns(table)
-            columns.each do |column|
-              if column.name.start_with?("encrypted_")
-                add_column = column.name
-                (h[table] ||= []) << add_column if add_column
+if defined?(ActiveRecord) && defined?(SyncAttr)
+  require "active_record"
+  require "sync_attr"
+
+  module RocketJob
+    module Jobs
+      module ReEncrypt
+        class RelationalJob < RocketJob::Job
+          include RocketJob::Batch
+
+          self.priority                 = 30
+          self.destroy_on_complete      = false
+          self.throttle_running_jobs    = 1
+          self.throttle_running_workers = 10
+
+          input_category slice_size: 1_000
+
+          # Name of the table being re-encrypted
+          field :table_name, type: String
+
+          # Limit the number of records to re-encrypt in test environments
+          field :limit, type: Integer
+
+          validates_presence_of :table_name
+          before_batch :upload_records
+
+          # Returns [Hash] of table names with each entry being an array
+          # of columns that start with encrypted_
+          sync_cattr_reader :encrypted_columns do
+            h = {}
+            connection.tables.each do |table|
+              columns = connection.columns(table)
+              columns.each do |column|
+                if column.name.start_with?("encrypted_")
+                  add_column = column.name
+                  (h[table] ||= []) << add_column if add_column
+                end
               end
             end
+            h
           end
-          h
-        end
 
-        # Re-encrypt all `encrypted_` columns in the relational database.
-        # Queues a Job for each table that needs re-encryption.
-        def self.start(**args)
-          encrypted_columns.keys.collect do |table|
-            create!(table_name: table, description: table, **args)
+          # Re-encrypt all `encrypted_` columns in the relational database.
+          # Queues a Job for each table that needs re-encryption.
+          def self.start(**args)
+            encrypted_columns.keys.collect do |table|
+              create!(table_name: table, description: table, **args)
+            end
           end
-        end
 
-        # Re-encrypt all encrypted columns for the named table.
-        # Does not use AR models since we do not have models for all tables.
-        def perform(range)
-          start_id, end_id = range
+          # Re-encrypt all encrypted columns for the named table.
+          # Does not use AR models since we do not have models for all tables.
+          def perform(range)
+            start_id, end_id = range
 
-          columns = self.class.encrypted_columns[table_name]
-          unless columns&.size&.positive?
-            logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
-            return
-          end
+            columns = self.class.encrypted_columns[table_name]
+            unless columns&.size&.positive?
+              logger.error "No columns for table: #{table_name} from #{start_id} to #{end_id}"
+              return
+            end
 
-          logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
-          sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
-
-          # Use AR to fetch all the records
-          self.class.connection.select_rows(sql).each do |row|
-            row     = row.unshift(nil)
-            index   = 1
-            sql     = "update #{quoted_table_name} set "
-            updates = []
-            columns.collect do |column|
-              index += 1
-              value = row[index]
-              # Prevent re-encryption
-              unless value.blank?
-                new_value = re_encrypt(value)
-                updates << "#{column} = \"#{new_value}\"" if new_value != value
+            logger.info "Processing: #{table_name} from #{start_id} to #{end_id}"
+            sql = "select id, #{columns.join(',')} from #{quoted_table_name} where id >= #{start_id} and id <= #{end_id}"
+
+            # Use AR to fetch all the records
+            self.class.connection.select_rows(sql).each do |row|
+              row.unshift(nil)
+              index   = 1
+              sql     = "update #{quoted_table_name} set "
+              updates = []
+              columns.collect do |column|
+                index += 1
+                value = row[index]
+                # Prevent re-encryption
+                unless value.blank?
+                  new_value = re_encrypt(value)
+                  updates << "#{column} = \"#{new_value}\"" if new_value != value
+                end
+              end
+              if updates.size.positive?
+                sql << updates.join(", ")
+                sql << " where id=#{row[1]}"
+                logger.trace sql
+                self.class.connection.execute sql
+              else
+                logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
               end
-            end
-            if updates.size.positive?
-              sql << updates.join(", ")
-              sql << " where id=#{row[1]}"
-              logger.trace sql
-              self.class.connection.execute sql
-            else
-              logger.trace { "Skipping empty values #{table_name}:#{row[1]}" }
             end
           end
-        end
 
-        # Returns a database connection.
-        #
-        # Override this method to support other ways of obtaining a thread specific database connection.
-        def self.connection
-          ActiveRecord::Base.connection
-        end
+          # Returns a database connection.
+          #
+          # Override this method to support other ways of obtaining a thread specific database connection.
+          def self.connection
+            ActiveRecord::Base.connection
+          end
 
-        private
+          private
 
-        def quoted_table_name
-          @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
-        end
+          def quoted_table_name
+            @quoted_table_name ||= self.class.connection.quote_table_name(table_name)
+          end
 
-        def re_encrypt(encrypted_value)
-          return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
+          def re_encrypt(encrypted_value)
+            return encrypted_value if (encrypted_value == "") || encrypted_value.nil?
 
-          SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
-        end
+            SymmetricEncryption.encrypt(SymmetricEncryption.decrypt(encrypted_value))
+          end
 
-        # Upload range to re-encrypt all rows in the specified table.
-        def upload_records
-          start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
-          last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
-          self.record_count = last_id.positive? ? (input.upload_integer_range_in_reverse_order(start_id, last_id) * slice_size) : 0
+          # Upload range to re-encrypt all rows in the specified table.
+          def upload_records
+            start_id          = self.class.connection.select_value("select min(id) from #{quoted_table_name}").to_i
+            last_id           = self.class.connection.select_value("select max(id) from #{quoted_table_name}").to_i
+            self.record_count =
+              if last_id.positive?
+                input.upload_integer_range_in_reverse_order(start_id, last_id) * input_category.slice_size
+              else
+                0
+              end
+          end
         end
       end
     end

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -19,7 +19,7 @@ module RocketJob
       field :properties, type: Hash, default: {}, user_editable: true
 
       # File to upload
-      field :upload_file_name, type: String, user_editable: true
+      field :upload_file_name, type: IOStreams::Path, user_editable: true
 
       # The original Input file name.
       # Used by #upload to extract the IOStreams when present.
@@ -33,10 +33,11 @@ module RocketJob
       validate :job_is_a_rocket_job
       validate :job_implements_upload
       validate :file_exists
+      validate :job_has_properties
 
       # Create the job and upload the file into it.
       def perform
-        job    = job_class.new(properties)
+        job    = job_class.from_properties(properties)
         job.id = job_id if job_id
         upload_file(job)
         job.save!
@@ -56,6 +57,10 @@ module RocketJob
 
       def upload_file(job)
         if job.respond_to?(:upload)
+          # Return the database connection for this thread back to the connection pool
+          # in case the upload takes a long time and the database connection expires.
+          ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
+
           if original_file_name
             job.upload(upload_file_name, file_name: original_file_name)
           else
@@ -66,7 +71,10 @@ module RocketJob
         elsif job.respond_to?(:full_file_name=)
           job.full_file_name = upload_file_name
         else
-          raise(ArgumentError, "Model #{job_class_name} must implement '#upload', or have attribute 'upload_file_name' or 'full_file_name'")
+          raise(
+            ArgumentError,
+            "Model #{job_class_name} must implement '#upload', or have attribute 'upload_file_name' or 'full_file_name'"
+          )
         end
       end
 
@@ -85,17 +93,49 @@ module RocketJob
         klass = job_class
         return if klass.nil? || klass.instance_methods.any? { |m| VALID_INSTANCE_METHODS.include?(m) }
 
-        errors.add(:job_class_name, "#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
+        errors.add(:job_class_name,
+                   "#{job_class} must implement any one of: :#{VALID_INSTANCE_METHODS.join(' :')} instance methods")
       end
 
       def file_exists
-        return if upload_file_name.nil?
+        # Only check for file existence when it is a local file
+        return unless upload_file_name.is_a?(IOStreams::Paths::File)
+        return errors.add(:upload_file_name, "Upload file name can't be blank.") if upload_file_name.to_s == ""
 
-        uri = URI.parse(upload_file_name)
-        return unless uri.scheme.nil? || uri.scheme == "file"
-        return if File.exist?(upload_file_name)
+        return if upload_file_name.exist?
 
         errors.add(:upload_file_name, "Upload file: #{upload_file_name} does not exist.")
+      rescue NotImplementedError
+        nil
+      end
+
+      def job_has_properties
+        klass = job_class
+        return unless klass
+
+        properties.each_pair do |k, _v|
+          next if klass.public_method_defined?("#{k}=".to_sym)
+
+          if %i[output_categories input_categories].include?(k)
+            category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
+            properties[k].each do |category|
+              category.each_pair do |key, _value|
+                next if category_class.public_method_defined?("#{key}=".to_sym)
+
+                errors.add(
+                  :properties,
+                  "Unknown Property in #{k}: Attempted to set a value for #{key}.#{k} which is not allowed on the job #{job_class_name}"
+                )
+              end
+            end
+            next
+          end
+
+          errors.add(
+            :properties,
+            "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}"
+          )
+        end
       end
     end
   end