maintenance_tasks 1.0.0

data/app/helpers/maintenance_tasks/application_helper.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+module MaintenanceTasks
+  # Module for common view helpers.
+  #
+  # @api private
+  module ApplicationHelper
+    include Pagy::Frontend
+
+    # Renders pagination for the page, if there is more than one page present.
+    #
+    # @param pagy [Pagy] the pagy instance containing pagination details,
+    #   including the number of pages the results are spread across.
+    # @return [String] the HTML to render for pagination.
+    def pagination(pagy)
+      raw(pagy_bulma_nav(pagy)) if pagy.pages > 1
+    end
+
+    # Renders a time element with the given datetime, worded as relative to the
+    # current time.
+    #
+    # The ISO 8601 version of the datetime is shown on hover
+    # via a title attribute.
+    #
+    # @param datetime [ActiveSupport::TimeWithZone] the time to be presented.
+    # @return [String] the HTML to render with the relative datetime in words.
+    def time_ago(datetime)
+      time_tag(datetime, title: datetime.utc.iso8601, class: 'is-clickable') do
+        time_ago_in_words(datetime) + ' ago'
+      end
+    end
+
+    # Fix stylesheet_link_tag to handle integrity when preloading.
+    # To be reverted once fixed upstream in Rails.
+    def stylesheet_link_tag(*sources)
+      return super unless respond_to?(:send_preload_links_header, true)
+      options = sources.extract_options!.stringify_keys
+      path_options = options.extract!('protocol', 'host', 'skip_pipeline')
+        .symbolize_keys
+      preload_links = []
+      crossorigin = options.delete('crossorigin')
+      crossorigin = 'anonymous' if crossorigin == true
+      nopush = options['nopush'].nil? ? true : options.delete('nopush')
+      integrity = options['integrity']
+
+      sources_tags = sources.uniq.map do |source|
+        href = path_to_stylesheet(source, path_options)
+        preload_link = "<#{href}>; rel=preload; as=style"
+        preload_link += "; crossorigin=#{crossorigin}" unless crossorigin.nil?
+        preload_link += "; integrity=#{integrity}" unless integrity.nil?
+        preload_link += '; nopush' if nopush
+        preload_links << preload_link
+        tag_options = {
+          'rel' => 'stylesheet',
+          'media' => 'screen',
+          'crossorigin' => crossorigin,
+          'href' => href,
+        }.merge!(options)
+        tag(:link, tag_options)
+      end
+
+      send_preload_links_header(preload_links)
+
+      safe_join(sources_tags)
+    end
+  end
+  private_constant :ApplicationHelper
+end

data/app/helpers/maintenance_tasks/task_helper.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'ripper'
+
+module MaintenanceTasks
+  # Helpers for formatting data in the maintenance_tasks views.
+  #
+  # @api private
+  module TaskHelper
+    STATUS_COLOURS = {
+      'new' => ['is-primary'],
+      'enqueued' => ['is-primary is-light'],
+      'running' => ['is-info'],
+      'interrupted' => ['is-info', 'is-light'],
+      'pausing' => ['is-warning', 'is-light'],
+      'paused' => ['is-warning'],
+      'succeeded' => ['is-success'],
+      'cancelling' => ['is-light'],
+      'cancelled' => ['is-dark'],
+      'errored' => ['is-danger'],
+    }
+
+    # Formats a run backtrace.
+    #
+    # @param backtrace [Array<String>] the backtrace associated with an
+    #   exception on a Task that ran and raised.
+    # @return [String] the parsed, HTML formatted version of the backtrace.
+    def format_backtrace(backtrace)
+      safe_join(backtrace.to_a, tag.br)
+    end
+
+    # Renders the progress bar.
+    #
+    # The style of the progress tag depends on the Run status. It also renders
+    # an infinite progress when a Run is active but there is no total
+    # information to estimate completion.
+    #
+    # @param run [Run] the Run which the progress bar will be based on.
+    #
+    # @return [String] the progress information properly formatted.
+    # @return [nil] if the run has not started yet.
+    def progress(run)
+      return unless run.started?
+
+      progress = Progress.new(run)
+
+      tag.progress(
+        value: progress.value,
+        max: progress.max,
+        title: progress.title,
+        class: ['progress'] + STATUS_COLOURS.fetch(run.status)
+      )
+    end
+
+    # Renders a span with a Run's status, with the corresponding tag class
+    # attached.
+    #
+    # @param status [String] the status for the Run.
+    # @return [String] the span element containing the status, with the
+    #   appropriate tag class attached.
+    def status_tag(status)
+      tag.span(status.capitalize, class: ['tag'] + STATUS_COLOURS.fetch(status))
+    end
+
+    # Returns the distance between now and the Run's expected completion time,
+    # if the Run has an estimated_completion_time.
+    #
+    # @param run [MaintenanceTasks::Run] the Run for which the estimated time to
+    #   completion is being calculated.
+    # return [String, nil] the distance in words, or nil if the Run has no
+    #   estimated completion time.
+    def estimated_time_to_completion(run)
+      estimated_completion_time = run.estimated_completion_time
+      if estimated_completion_time.present?
+        time_ago_in_words(estimated_completion_time)
+      end
+    end
+
+    # Reports the approximate elapsed time a Run has been processed so far based
+    # on the Run's time running attribute.
+    #
+    # @param run [Run] the source of the time to be reported.
+    #
+    # @return [String] the description of the time running attribute.
+    def time_running_in_words(run)
+      distance_of_time_in_words(0, run.time_running, include_seconds: true)
+    end
+
+    # Very simple syntax highlighter based on Ripper.
+    #
+    # It returns the same code except identifiers, keywords, etc. are wrapped
+    # in +<span>+ tags with CSS classes that match the types returned by
+    # Ripper.lex.
+    #
+    # @param code [String] the Ruby code source to syntax highlight.
+    # @return [ActiveSupport::SafeBuffer] HTML of the code.
+    def highlight_code(code)
+      tokens = Ripper.lex(code).map do |(_position, type, content, _state)|
+        case type
+        when :on_nl, :on_sp, :on_ignored_nl
+          content
+        else
+          tag.span(content, class: type.to_s.sub('on_', 'ruby-').dasherize)
+        end
+      end
+      safe_join(tokens)
+    end
+  end
+  private_constant :TaskHelper
+end

data/app/jobs/maintenance_tasks/task_job.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module MaintenanceTasks
+  # Base class that is inherited by the host application's task classes.
+  class TaskJob < ActiveJob::Base
+    include JobIteration::Iteration
+
+    before_perform(:before_perform)
+
+    on_start(:on_start)
+    on_complete(:on_complete)
+    on_shutdown(:on_shutdown)
+
+    after_perform(:after_perform)
+
+    rescue_from StandardError, with: :on_error
+
+    class << self
+      # Overrides ActiveJob::Exceptions.retry_on to declare it unsupported.
+      # The use of rescue_from prevents retry_on from being usable.
+      def retry_on(*, **)
+        raise NotImplementedError, 'retry_on is not supported'
+      end
+    end
+
+    private
+
+    def build_enumerator(_run, cursor:)
+      cursor ||= @run.cursor
+      collection = @task.collection
+
+      case collection
+      when ActiveRecord::Relation
+        enumerator_builder.active_record_on_records(collection, cursor: cursor)
+      when Array
+        enumerator_builder.build_array_enumerator(collection, cursor: cursor)
+      else
+        raise ArgumentError, "#{@task.class.name}#collection must be either "\
+          'an Active Record Relation or an Array.'
+      end
+    end
+
+    # Performs task iteration logic for the current input returned by the
+    # enumerator.
+    #
+    # @param input [Object] the current element from the enumerator.
+    # @param _run [Run] the current Run, passed as an argument by Job Iteration.
+    def each_iteration(input, _run)
+      throw(:abort, :skip_complete_callbacks) if @run.stopping?
+      @task.process(input)
+      @ticker.tick
+      @run.reload_status
+    end
+
+    def before_perform
+      @run = arguments.first
+      @task = Task.named(@run.task_name).new
+      @run.job_id = job_id
+
+      @run.running! unless @run.stopping?
+
+      @ticker = Ticker.new(MaintenanceTasks.ticker_delay) do |ticks, duration|
+        @run.persist_progress(ticks, duration)
+      end
+    end
+
+    def on_start
+      @run.update!(started_at: Time.now, tick_total: @task.count)
+    end
+
+    def on_complete
+      @run.status = :succeeded
+      @run.ended_at = Time.now
+    end
+
+    def on_shutdown
+      if @run.cancelling?
+        @run.status = :cancelled
+        @run.ended_at = Time.now
+      else
+        @run.status = @run.pausing? ? :paused : :interrupted
+        @run.cursor = cursor_position
+      end
+
+      @ticker.persist
+    end
+
+    def after_perform
+      @run.save!
+    end
+
+    def on_error(error)
+      @ticker.persist
+      @run.persist_error(error)
+      MaintenanceTasks.error_handler.call(error)
+    end
+  end
+end

data/app/models/maintenance_tasks/application_record.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+module MaintenanceTasks
+  # Base class for all records used by this engine.
+  #
+  # @api private
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+  private_constant :ApplicationRecord
+end

data/app/models/maintenance_tasks/progress.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module MaintenanceTasks
+  # This class generates progress information about a Run.
+  class Progress
+    include ActiveSupport::NumberHelper
+
+    # Sets the Progress initial state with a Run.
+    #
+    # @param run [Run] the source of progress information.
+    def initialize(run)
+      @run = run
+    end
+
+    # Defines the value of progress information. This represents the amount that
+    # is already done out of the progress maximum.
+    #
+    # For indefinite-style progress information, value is nil. That highlights
+    # that a Run is in progress but it is not possible to estimate how close to
+    # completion it is.
+    #
+    # When a Run is stopped, the value is present even if there is no total.
+    # That represents a progress information that assumes that the current value
+    # is also equal to is max, showing a progress as completed.
+    #
+    # @return [Integer] if progress can be determined or the Run is stopped.
+    # @return [nil] if progress can't be determined and the Run isn't stopped.
+    def value
+      @run.tick_count if estimatable? || @run.stopped?
+    end
+
+    # The maximum amount of work expected to be done. This is extracted from the
+    # Run's tick total attribute when present, or it is equal to the Run's
+    # tick count.
+    #
+    # This amount is enqual to the Run's tick count if the tick count is greater
+    # than the tick total. This represents that the total was underestimated.
+    #
+    # @return [Integer] the progress maximum amount.
+    def max
+      estimatable? ? @run.tick_total : @run.tick_count
+    end
+
+    # The title for the progress information. This is a text that describes the
+    # progress of the Run so far. It includes the percentage that is done out of
+    # the maximum, if an estimate is possible.
+    #
+    # @return [String] the title for the Run progress.
+    def title
+      if !total?
+        "Processed #{@run.tick_count} #{'item'.pluralize(@run.tick_count)}."
+      elsif @run.tick_count > @run.tick_total
+        "Processed #{@run.tick_count} #{'item'.pluralize(@run.tick_count)} " \
+          "(expected #{@run.tick_total})."
+      else
+        percentage = 100.0 * @run.tick_count / @run.tick_total
+
+        "Processed #{@run.tick_count} out of #{@run.tick_total} "\
+          "(#{number_to_percentage(percentage, precision: 0)})"
+      end
+    end
+
+    private
+
+    def total?
+      @run.tick_total.to_i > 0
+    end
+
+    def estimatable?
+      total? && @run.tick_total > @run.tick_count
+    end
+  end
+  private_constant :Progress
+end

data/app/models/maintenance_tasks/run.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+module MaintenanceTasks
+  # Model that persists information related to a task being run from the UI.
+  #
+  # @api private
+  class Run < ApplicationRecord
+    # Various statuses a run can be in.
+    STATUSES = [
+      :enqueued,    # The task has been enqueued by the user.
+      :running,     # The task is being performed by a job worker.
+      :succeeded,   # The task finished without error.
+      :cancelling,  # The task has been told to cancel but is finishing work.
+      :cancelled,   # The user explicitly halted the task's execution.
+      :interrupted, # The task was interrupted by the job infrastructure.
+      :pausing,     # The task has been told to pause but is finishing work.
+      :paused,      # The task was paused in the middle of the run by the user.
+      :errored,     # The task code produced an unhandled exception.
+    ]
+
+    ACTIVE_STATUSES = [
+      :enqueued,
+      :running,
+      :paused,
+      :pausing,
+      :cancelling,
+      :interrupted,
+    ]
+    COMPLETED_STATUSES = [:succeeded, :errored, :cancelled]
+    COMPLETED_RUNS_LIMIT = 10
+
+    enum status: STATUSES.to_h { |status| [status, status.to_s] }
+
+    validates :task_name, on: :create, inclusion: { in: ->(_) {
+      Task.available_tasks.map(&:to_s)
+    } }
+    attr_readonly :task_name
+
+    serialize :backtrace
+
+    scope :active, -> { where(status: ACTIVE_STATUSES) }
+
+    validates_with RunStatusValidator, on: :update
+
+    # Sets the run status to enqueued, making sure the transition is validated
+    # in case it's already enqueued.
+    def enqueued!
+      status_will_change!
+      super
+    end
+
+    # Increments +tick_count+ by +number_of_ticks+ and +time_running+ by
+    # +duration+, both directly in the DB.
+    # The attribute values are not set in the current instance, you need
+    # to reload the record.
+    #
+    # @param number_of_ticks [Integer] number of ticks to add to tick_count.
+    # @param duration [Float] the time in seconds that elapsed since the last
+    #   increment of ticks.
+    def persist_progress(number_of_ticks, duration)
+      self.class.update_counters(
+        id,
+        tick_count: number_of_ticks,
+        time_running: duration,
+        touch: true
+      )
+    end
+
+    # Marks the run as errored and persists the error data.
+    #
+    # @param error [StandardError] the Error being persisted.
+    def persist_error(error)
+      update!(
+        status: :errored,
+        error_class: error.class.to_s,
+        error_message: error.message,
+        backtrace: Rails.backtrace_cleaner.clean(error.backtrace),
+        ended_at: Time.now,
+      )
+    end
+
+    # Refreshes just the status attribute on the Active Record object, and
+    # ensures ActiveModel::Dirty does not mark the object as changed.
+    # This allows us to get the Run's most up-to-date status without needing
+    # to reload the entire record.
+    #
+    # @return [MaintenanceTasks::Run] the Run record with its updated status.
+    def reload_status
+      updated_status = Run.uncached do
+        Run.where(id: id).pluck(:status).first
+      end
+      self.status = updated_status
+      clear_attribute_changes([:status])
+      self
+    end
+
+    # Returns whether the Run is stopping, which is defined as
+    # having a status of pausing or cancelled.
+    #
+    # @return [Boolean] whether the Run is stopping.
+    def stopping?
+      pausing? || cancelling?
+    end
+
+    # Returns whether the Run is stopped, which is defined as having a status of
+    # paused, succeeded, cancelled, or errored.
+    #
+    # @return [Boolean] whether the Run is stopped.
+    def stopped?
+      completed? || paused?
+    end
+
+    # Returns whether the Run has been started, which is indicated by the
+    # started_at timestamp being present.
+    #
+    # @return [Boolean] whether the Run was started.
+    def started?
+      started_at.present?
+    end
+
+    # Returns whether the Run is completed, which is defined as
+    # having a status of succeeded, cancelled, or errored.
+    #
+    # @return [Boolean] whether the Run is completed.
+    def completed?
+      COMPLETED_STATUSES.include?(status.to_sym)
+    end
+
+    # Returns whether the Run is active, which is defined as
+    # having a status of enqueued, running, pausing, cancelling,
+    # paused or interrupted.
+    #
+    # @return [Boolean] whether the Run is active.
+    def active?
+      ACTIVE_STATUSES.include?(status.to_sym)
+    end
+
+    # Returns the estimated time the task will finish based on the the number of
+    # ticks left and the average time needed to process a tick.
+    # Returns nil if the Run is completed, or if the tick_count or tick_total is
+    # zero.
+    #
+    # @return [Time] the estimated time the Run will finish.
+    def estimated_completion_time
+      return if completed? || tick_count == 0 || tick_total.to_i == 0
+
+      processed_per_second = (tick_count.to_f / time_running)
+      ticks_left = (tick_total - tick_count)
+      seconds_to_finished = ticks_left / processed_per_second
+      Time.now + seconds_to_finished
+    end
+
+    # Cancels a Run.
+    #
+    # If the Run is paused, it will transition directly to cancelled, since the
+    # Task is not being performed. In this case, the ended_at timestamp
+    # will be updated.
+    #
+    # If the Run is not paused, the Run will transition to cancelling.
+    #
+    # If the Run is already cancelling, and has last been updated more than 5
+    # minutes ago, it will transition to cancelled, and the ended_at timestamp
+    # will be updated.
+    def cancel
+      if paused? || stuck?
+        update!(status: :cancelled, ended_at: Time.now)
+      else
+        cancelling!
+      end
+    end
+
+    # Returns whether a Run is stuck, which is defined as having a status of
+    # cancelling, and not having been updated in the last 5 minutes.
+    #
+    # @return [Boolean] whether the Run is stuck.
+    def stuck?
+      cancelling? && updated_at <= 5.minutes.ago
+    end
+  end
+  private_constant :Run
+end