maintenance_tasks 1.3.0 → 1.7.0

data/app/jobs/concerns/maintenance_tasks/task_job_concern.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module MaintenanceTasks
   # Concern that holds the behaviour of the job that runs the tasks. It is
   # included in {TaskJob} and if MaintenanceTasks.job is overridden, it must be
@@ -11,8 +12,8 @@ module MaintenanceTasks
       before_perform(:before_perform)
 
       on_start(:on_start)
-      on_complete(:on_complete)
       on_shutdown(:on_shutdown)
+      on_complete(:on_complete)
 
       after_perform(:after_perform)
 
@@ -32,21 +33,46 @@ module MaintenanceTasks
     def build_enumerator(_run, cursor:)
       cursor ||= @run.cursor
       collection = @task.collection
+      @enumerator = nil
 
       collection_enum = case collection
       when ActiveRecord::Relation
         enumerator_builder.active_record_on_records(collection, cursor: cursor)
+      when ActiveRecord::Batches::BatchEnumerator
+        if collection.start || collection.finish
+          raise ArgumentError, <<~MSG.squish
+            #{@task.class.name}#collection cannot support
+            a batch enumerator with the "start" or "finish" options.
+          MSG
+        end
+        # For now, only support automatic count based on the enumerator for
+        # batches
+        @enumerator = enumerator_builder.active_record_on_batch_relations(
+          collection.relation,
+          cursor: cursor,
+          batch_size: collection.batch_size,
+        )
       when Array
         enumerator_builder.build_array_enumerator(collection, cursor: cursor)
       when CSV
         JobIteration::CsvEnumerator.new(collection).rows(cursor: cursor)
       else
-        raise ArgumentError, "#{@task.class.name}#collection must be either "\
-          "an Active Record Relation, Array, or CSV."
+        raise ArgumentError, <<~MSG.squish
+          #{@task.class.name}#collection must be either an
+          Active Record Relation, ActiveRecord::Batches::BatchEnumerator,
+          Array, or CSV.
+        MSG
       end
+      throttle_enumerator(collection_enum)
+    end
 
+    def throttle_enumerator(collection_enum)
       @task.throttle_conditions.reduce(collection_enum) do |enum, condition|
-        enumerator_builder.build_throttle_enumerator(enum, **condition)
+        enumerator_builder.build_throttle_enumerator(
+          enum,
+          throttle_on: condition[:throttle_on],
+          backoff: condition[:backoff].call
+        )
       end
     end
 
@@ -71,13 +97,12 @@ module MaintenanceTasks
 
     def before_perform
       @run = arguments.first
-      @task = Task.named(@run.task_name).new
-      if @task.respond_to?(:csv_content=)
+      @task = @run.task
+      if @task.has_csv_content?
         @task.csv_content = @run.csv_file.download
       end
-      @run.job_id = job_id
 
-      @run.running! unless @run.stopping?
+      @run.running
 
       @ticker = Ticker.new(MaintenanceTasks.ticker_delay) do |ticks, duration|
         @run.persist_progress(ticks, duration)
@@ -85,26 +110,21 @@ module MaintenanceTasks
     end
 
     def on_start
-      @run.update!(started_at: Time.now, tick_total: @task.count)
-    end
-
-    def on_complete
-      @run.status = :succeeded
-      @run.ended_at = Time.now
+      count = @task.count
+      count = @enumerator&.size if count == :no_count
+      @run.start(count)
     end
 
     def on_shutdown
-      if @run.cancelling?
-        @run.status = :cancelled
-        @run.ended_at = Time.now
-      else
-        @run.status = @run.pausing? ? :paused : :interrupted
-        @run.cursor = cursor_position
-      end
-
+      @run.job_shutdown
+      @run.cursor = cursor_position
       @ticker.persist
     end
 
+    def on_complete
+      @run.complete
+    end
+
     # We are reopening a private part of Job Iteration's API here, so we should
     # ensure the method is still defined upstream. This way, in the case where
     # the method changes upstream, we catch it at load time instead of at
@@ -123,7 +143,7 @@ module MaintenanceTasks
     end
 
     def after_perform
-      @run.save!
+      @run.persist_transition
       if defined?(@reenqueue_iteration_job) && @reenqueue_iteration_job
         reenqueue_iteration_job(should_ignore: false)
       end
@@ -144,6 +164,7 @@ module MaintenanceTasks
         task_context = {}
       end
       errored_element = @errored_element if defined?(@errored_element)
+    ensure
       MaintenanceTasks.error_handler.call(error, task_context, errored_element)
     end
   end

data/app/models/maintenance_tasks/application_record.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module MaintenanceTasks
   # Base class for all records used by this engine.
   #

data/app/models/maintenance_tasks/csv_collection_builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module MaintenanceTasks
+  # Strategy for building a Task that processes CSV files.
+  #
+  # @api private
+  class CsvCollectionBuilder
+    # Defines the collection to be iterated over, based on the provided CSV.
+    #
+    # @return [CSV] the CSV object constructed from the specified CSV content,
+    #   with headers.
+    def collection(task)
+      CSV.new(task.csv_content, headers: true)
+    end
+
+    # The number of rows to be processed. Excludes the header row from the
+    # count and assumes a trailing newline is at the end of the CSV file.
+    # Note that this number is an approximation based on the number of
+    # newlines.
+    #
+    # @return [Integer] the approximate number of rows to process.
+    def count(task)
+      task.csv_content.count("\n") - 1
+    end
+
+    # Return that the Task processes CSV content.
+    def has_csv_content?
+      true
+    end
+  end
+end

data/app/models/maintenance_tasks/null_collection_builder.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module MaintenanceTasks
+  # Base strategy for building a collection-based Task to be performed.
+  class NullCollectionBuilder
+    # Placeholder method to raise in case a subclass fails to implement the
+    # expected instance method.
+    #
+    # @raise [NotImplementedError] with a message advising subclasses to
+    #   implement an override for this method.
+    def collection(task)
+      raise NoMethodError, "#{task.class.name} must implement `collection`."
+    end
+
+    # Total count of iterations to be performed.
+    #
+    # Tasks override this method to define the total amount of iterations
+    # expected at the start of the run. Return +nil+ if the amount is
+    # undefined, or counting would be prohibitive for your database.
+    #
+    # @return [Integer, nil]
+    def count(task)
+      :no_count
+    end
+
+    # Return that the Task does not process CSV content.
+    def has_csv_content?
+      false
+    end
+  end
+end

data/app/models/maintenance_tasks/progress.rb

@@ -50,14 +50,19 @@ module MaintenanceTasks
     def text
       count = @run.tick_count
       total = @run.tick_total
+
       if !total?
-        "Processed #{pluralize(count, "item")}."
+        "Processed #{number_to_delimited(count)} "\
+          "#{"item".pluralize(count)}."
       elsif over_total?
-        "Processed #{pluralize(count, "item")} (expected #{total})."
+        "Processed #{number_to_delimited(count)} "\
+          "#{"item".pluralize(count)} "\
+          "(expected #{number_to_delimited(total)})."
       else
         percentage = 100.0 * count / total
 
-        "Processed #{count} out of #{pluralize(total, "item")} "\
+        "Processed #{number_to_delimited(count)} out of "\
+          "#{number_to_delimited(total)} #{"item".pluralize(total)} "\
           "(#{number_to_percentage(percentage, precision: 0)})."
       end
     end

data/app/models/maintenance_tasks/run.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module MaintenanceTasks
   # Model that persists information related to a task being run from the UI.
   #
@@ -25,8 +26,13 @@ module MaintenanceTasks
       :cancelling,
       :interrupted,
     ]
+    STOPPING_STATUSES = [
+      :pausing,
+      :cancelling,
+    ]
     COMPLETED_STATUSES = [:succeeded, :errored, :cancelled]
     COMPLETED_RUNS_LIMIT = 10
+    STUCK_TASK_TIMEOUT = 5.minutes
 
     enum status: STATUSES.to_h { |status| [status, status.to_s] }
 
@@ -34,10 +40,12 @@ module MaintenanceTasks
       Task.available_tasks.map(&:to_s)
     } }
     validate :csv_attachment_presence, on: :create
+    validate :validate_task_arguments, on: :create
 
     attr_readonly :task_name
 
     serialize :backtrace
+    serialize :arguments, JSON
 
     scope :active, -> { where(status: ACTIVE_STATUSES) }
 
@@ -58,9 +66,40 @@ module MaintenanceTasks
 
     # Sets the run status to enqueued, making sure the transition is validated
     # in case it's already enqueued.
+    #
+    # Rescues and retries status transition if an ActiveRecord::StaleObjectError
+    # is encountered.
     def enqueued!
       status_will_change!
       super
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
+    end
+
+    CALLBACKS_TRANSITION = {
+      cancelled: :cancel,
+      interrupted: :interrupt,
+      paused: :pause,
+      succeeded: :complete,
+    }.transform_keys(&:to_s)
+    private_constant :CALLBACKS_TRANSITION
+
+    # Saves the run, persisting the transition of its status, and all other
+    # changes to the object.
+    def persist_transition
+      save!
+      callback = CALLBACKS_TRANSITION[status]
+      run_task_callbacks(callback) if callback
+    rescue ActiveRecord::StaleObjectError
+      success = succeeded?
+      reload_status
+      if success
+        self.status = :succeeded
+      else
+        job_shutdown
+      end
+      retry
     end
 
     # Increments +tick_count+ by +number_of_ticks+ and +time_running+ by
@@ -78,33 +117,53 @@ module MaintenanceTasks
         time_running: duration,
         touch: true
       )
+      if locking_enabled?
+        locking_column = self.class.locking_column
+        self[locking_column] += 1
+        clear_attribute_change(locking_column)
+      end
     end
 
     # Marks the run as errored and persists the error data.
     #
     # @param error [StandardError] the Error being persisted.
     def persist_error(error)
+      self.started_at ||= Time.now
       update!(
         status: :errored,
         error_class: error.class.to_s,
         error_message: error.message,
-        backtrace: Rails.backtrace_cleaner.clean(error.backtrace),
+        backtrace: MaintenanceTasks.backtrace_cleaner.clean(error.backtrace),
         ended_at: Time.now,
       )
+      run_task_callbacks(:error)
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
     end
 
-    # Refreshes just the status attribute on the Active Record object, and
-    # ensures ActiveModel::Dirty does not mark the object as changed.
+    # Refreshes the status and lock version attributes on the Active Record
+    # object, and ensures ActiveModel::Dirty doesn't mark the object as changed.
+    #
     # This allows us to get the Run's most up-to-date status without needing
     # to reload the entire record.
     #
     # @return [MaintenanceTasks::Run] the Run record with its updated status.
     def reload_status
-      updated_status = Run.uncached do
-        Run.where(id: id).pluck(:status).first
+      columns_to_reload = if locking_enabled?
+        [:status, self.class.locking_column]
+      else
+        [:status]
       end
+      updated_status, updated_lock_version = self.class.uncached do
+        self.class.where(id: id).pluck(*columns_to_reload).first
+      end
+
       self.status = updated_status
-      clear_attribute_changes([:status])
+      if updated_lock_version
+        self[self.class.locking_column] = updated_lock_version
+      end
+      clear_attribute_changes(columns_to_reload)
       self
     end
 
@@ -113,7 +172,7 @@ module MaintenanceTasks
     #
     # @return [Boolean] whether the Run is stopping.
     def stopping?
-      pausing? || cancelling?
+      STOPPING_STATUSES.include?(status.to_sym)
     end
 
     # Returns whether the Run is stopped, which is defined as having a status of
@@ -149,19 +208,78 @@ module MaintenanceTasks
       ACTIVE_STATUSES.include?(status.to_sym)
     end
 
-    # Returns the estimated time the task will finish based on the the number of
-    # ticks left and the average time needed to process a tick.
-    # Returns nil if the Run is completed, or if the tick_count or tick_total is
-    # zero.
+    # Returns the duration left for the Run to finish based on the number of
+    # ticks left and the average time needed to process a tick. Returns nil if
+    # the Run is completed, or if tick_count or tick_total is zero.
     #
-    # @return [Time] the estimated time the Run will finish.
-    def estimated_completion_time
+    # @return [ActiveSupport::Duration] the estimated duration left for the Run
+    #   to finish.
+    def time_to_completion
       return if completed? || tick_count == 0 || tick_total.to_i == 0
 
       processed_per_second = (tick_count.to_f / time_running)
       ticks_left = (tick_total - tick_count)
       seconds_to_finished = ticks_left / processed_per_second
-      Time.now + seconds_to_finished
+      seconds_to_finished.seconds
+    end
+
+    # Marks a Run as running.
+    #
+    # If the run is stopping already, it will not transition to running.
+    # Rescues and retries status transition if an ActiveRecord::StaleObjectError
+    # is encountered.
+    def running
+      if locking_enabled?
+        begin
+          running! unless stopping?
+        rescue ActiveRecord::StaleObjectError
+          reload_status
+          retry
+        end
+      else
+        # Preserve swap-and-replace solution for data races until users
+        # run migration to upgrade to optimistic locking solution
+        return if stopping?
+        updated = self.class.where(id: id).where.not(status: STOPPING_STATUSES)
+          .update_all(status: :running, updated_at: Time.now) > 0
+        if updated
+          self.status = :running
+          clear_attribute_changes([:status])
+        else
+          reload_status
+        end
+      end
+    end
+
+    # Starts a Run, setting its started_at timestamp and tick_total.
+    #
+    # @param count [Integer] the total iterations to be performed, as
+    #   specified by the Task.
+    def start(count)
+      update!(started_at: Time.now, tick_total: count)
+      run_task_callbacks(:start)
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
+    end
+
+    # Handles transitioning the status on a Run when the job shuts down.
+    def job_shutdown
+      if cancelling?
+        self.status = :cancelled
+        self.ended_at = Time.now
+      elsif pausing?
+        self.status = :paused
+      else
+        self.status = :interrupted
+      end
+    end
+
+    # Handles the completion of a Run, setting a status of succeeded and the
+    # ended_at timestamp.
+    def complete
+      self.status = :succeeded
+      self.ended_at = Time.now
     end
 
     # Cancels a Run.
@@ -177,10 +295,26 @@ module MaintenanceTasks
     # will be updated.
     def cancel
       if paused? || stuck?
-        update!(status: :cancelled, ended_at: Time.now)
+        self.status = :cancelled
+        self.ended_at = Time.now
+        persist_transition
       else
         cancelling!
       end
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
+    end
+
+    # Marks a Run as pausing.
+    #
+    # Rescues and retries status transition if an ActiveRecord::StaleObjectError
+    # is encountered.
+    def pausing!
+      super
+    rescue ActiveRecord::StaleObjectError
+      reload_status
+      retry
     end
 
     # Returns whether a Run is stuck, which is defined as having a status of
@@ -188,7 +322,7 @@ module MaintenanceTasks
     #
     # @return [Boolean] whether the Run is stuck.
     def stuck?
-      cancelling? && updated_at <= 5.minutes.ago
+      cancelling? && updated_at <= STUCK_TASK_TIMEOUT.ago
     end
 
     # Performs validation on the presence of a :csv_file attachment.
@@ -197,15 +331,51 @@ module MaintenanceTasks
     # should not have an attachment to be valid. The appropriate error is added
     # if the Run does not meet the above criteria.
     def csv_attachment_presence
-      if Task.named(task_name) < CsvCollection && !csv_file.attached?
+      if Task.named(task_name).has_csv_content? && !csv_file.attached?
         errors.add(:csv_file, "must be attached to CSV Task.")
-      elsif !(Task.named(task_name) < CsvCollection) && csv_file.present?
+      elsif !Task.named(task_name).has_csv_content? && csv_file.present?
         errors.add(:csv_file, "should not be attached to non-CSV Task.")
       end
     rescue Task::NotFoundError
       nil
     end
 
+    # Support iterating over ActiveModel::Errors in Rails 6.0 and Rails 6.1+.
+    # To be removed when Rails 6.0 is no longer supported.
+    if Rails::VERSION::STRING.match?(/^6.0/)
+      # Performs validation on the arguments to use for the Task. If the Task is
+      # invalid, the errors are added to the Run.
+      def validate_task_arguments
+        arguments_match_task_attributes if arguments.present?
+        if task.invalid?
+          error_messages = task.errors
+            .map { |attribute, message| "#{attribute.inspect} #{message}" }
+          errors.add(
+            :arguments,
+            "are invalid: #{error_messages.join("; ")}"
+          )
+        end
+      rescue Task::NotFoundError
+        nil
+      end
+    else
+      # Performs validation on the arguments to use for the Task. If the Task is
+      # invalid, the errors are added to the Run.
+      def validate_task_arguments
+        arguments_match_task_attributes if arguments.present?
+        if task.invalid?
+          error_messages = task.errors
+            .map { |error| "#{error.attribute.inspect} #{error.message}" }
+          errors.add(
+            :arguments,
+            "are invalid: #{error_messages.join("; ")}"
+          )
+        end
+      rescue Task::NotFoundError
+        nil
+      end
+    end
+
     # Fetches the attached ActiveStorage CSV file for the run. Checks first
     # whether the ActiveStorage::Attachment table exists so that we are
     # compatible with apps that are not using ActiveStorage.
@@ -216,5 +386,41 @@ module MaintenanceTasks
       return unless ActiveStorage::Attachment.table_exists?
       super
     end
+
+    # Returns a Task instance for this Run. Assigns any attributes to the Task
+    # based on the Run's parameters. Note that the Task instance is not supplied
+    # with :csv_content yet if it's a CSV Task. This is done in the job, since
+    # downloading the CSV file can take some time.
+    #
+    # @return [Task] a Task instance.
+    def task
+      @task ||= begin
+        task = Task.named(task_name).new
+        if task.attribute_names.any? && arguments.present?
+          task.assign_attributes(arguments)
+        end
+        task
+      rescue ActiveModel::UnknownAttributeError
+        task
+      end
+    end
+
+    private
+
+    def run_task_callbacks(callback)
+      task.run_callbacks(callback)
+    rescue
+      nil
+    end
+
+    def arguments_match_task_attributes
+      invalid_argument_keys = arguments.keys - task.attribute_names
+      if invalid_argument_keys.any?
+        error_message = <<~MSG.squish
+          Unknown parameters: #{invalid_argument_keys.map(&:to_sym).join(", ")}
+        MSG
+        errors.add(:base, error_message)
+      end
+    end
   end
 end

data/app/models/maintenance_tasks/runner.rb

@@ -37,25 +37,35 @@ module MaintenanceTasks
     #   for the Task to iterate over when running, in the form of an attachable
     #   (see https://edgeapi.rubyonrails.org/classes/ActiveStorage/Attached/One.html#method-i-attach).
     #   Value is nil if the Task does not use CSV iteration.
+    # @param arguments [Hash] the arguments to persist to the Run and to make
+    #   accessible to the Task.
     #
     # @return [Task] the Task that was run.
     #
     # @raise [EnqueuingError] if an error occurs while enqueuing the Run.
     # @raise [ActiveRecord::RecordInvalid] if validation errors occur while
     #   creating the Run.
-    def run(name:, csv_file: nil)
-      run = Run.active.find_by(task_name: name) || Run.new(task_name: name)
-      run.csv_file.attach(csv_file) if csv_file
-
+    # @raise [ActiveRecord::ValueTooLong] if the creation of the Run fails due
+    #   to a value being too long for the column type.
+    def run(name:, csv_file: nil, arguments: {}, run_model: Run)
+      run = run_model.active.find_by(task_name: name) ||
+        run_model.new(task_name: name, arguments: arguments)
+      if csv_file
+        run.csv_file.attach(csv_file)
+        run.csv_file.filename = filename(name)
+      end
+      job = instantiate_job(run)
+      run.job_id = job.job_id
+      yield run if block_given?
       run.enqueued!
-      enqueue(run)
+      enqueue(run, job)
       Task.named(name)
     end
 
     private
 
-    def enqueue(run)
-      unless MaintenanceTasks.job.constantize.perform_later(run)
+    def enqueue(run, job)
+      unless job.enqueue
         raise "The job to perform #{run.task_name} could not be enqueued. "\
           "Enqueuing has been prevented by a callback."
       end
@@ -63,5 +73,14 @@ module MaintenanceTasks
       run.persist_error(error)
       raise EnqueuingError, run
     end
+
+    def filename(task_name)
+      formatted_task_name = task_name.underscore.gsub("/", "_")
+      "#{Time.now.utc.strftime("%Y%m%dT%H%M%SZ")}_#{formatted_task_name}.csv"
+    end
+
+    def instantiate_job(run)
+      MaintenanceTasks.job.constantize.new(run)
+    end
   end
 end

data/app/models/maintenance_tasks/runs_page.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module MaintenanceTasks
   # This class is responsible for handling cursor-based pagination for Run
   # records.