solid_job_monitor 1.0.0

data/app/assets/stylesheets/mission_control/jobs/forms.css

@@ -0,0 +1,8 @@
+/* Hide the HTML datalist wonky arrow in Safari and Chrome, since we add it via bulma. */
+[list]::-webkit-calendar-picker-indicator {
+    opacity: 0;
+}
+
+a[disabled] {
+    pointer-events: none;
+}

data/app/assets/stylesheets/mission_control/jobs/jobs.css

@@ -0,0 +1,68 @@
+.filter input {
+    width: 15rem;
+}
+
+table.jobs {
+    td {
+        word-break: break-all;
+    }
+    th.job-header{
+        width: 30%;
+    }
+    th.duration-header {
+        width: 20%;
+    }
+    &.queues th.job-header {
+        width: 30%;
+    }
+    &.failed th.job-header {
+        width: 35%;
+    }
+    &.in_progress th.job-header {
+        width: 35%;
+    }
+    &.blocked th.job-header {
+        width: 45%;
+    }
+    &.scheduled th.job-header {
+        width: 40%;
+    }
+    &.finished th.job-header {
+        width: 65%;
+    }
+    &.workers th.job-header {
+        width: 40%;
+    }
+}
+
+.job-progress-wrap {
+    min-width: 7rem;
+}
+
+.job-progress {
+    position: relative;
+    flex: 1;
+    height: 1.25rem;
+    min-width: 6rem;
+    background: #ededed;
+    border-radius: 4px;
+    overflow: hidden;
+}
+
+.job-progress-bar {
+    width: 0;
+    height: 100%;
+    background: #00d1b2;
+    transition: width 0.3s ease;
+}
+
+.job-progress-label {
+    position: absolute;
+    inset: 0;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-size: 0.75rem;
+    font-weight: 600;
+    color: #363636;
+}

data/app/controllers/concerns/mission_control/jobs/adapter_features.rb

@@ -0,0 +1,24 @@
+module MissionControl::Jobs::AdapterFeatures
+  extend ActiveSupport::Concern
+
+  included do
+    helper_method :supported_job_statuses, :queue_pausing_supported?, :workers_exposed?, :recurring_tasks_supported?
+  end
+
+  private
+    def supported_job_statuses
+      MissionControl::Jobs::Current.server.queue_adapter.supported_job_statuses & ActiveJob::JobsRelation::STATUSES
+    end
+
+    def queue_pausing_supported?
+      MissionControl::Jobs::Current.server.queue_adapter.supports_queue_pausing?
+    end
+
+    def workers_exposed?
+      MissionControl::Jobs::Current.server.queue_adapter.exposes_workers?
+    end
+
+    def recurring_tasks_supported?
+      MissionControl::Jobs::Current.server.queue_adapter.supports_recurring_tasks?
+    end
+end

data/app/controllers/concerns/mission_control/jobs/application_scoped.rb

@@ -0,0 +1,38 @@
+module MissionControl::Jobs::ApplicationScoped
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :set_application
+    around_action :activating_job_server
+
+    delegate :applications, to: MissionControl::Jobs
+  end
+
+  private
+    def set_application
+      @application = find_application or raise MissionControl::Jobs::Errors::ResourceNotFound, "Application not found"
+      MissionControl::Jobs::Current.application = @application
+    end
+
+    def find_application
+      if params[:application_id]
+        applications[params[:application_id]]
+      else
+        applications.first
+      end
+    end
+
+    def activating_job_server(&block)
+      @server = find_server or raise MissionControl::Jobs::Errors::ResourceNotFound, "Server not found"
+      MissionControl::Jobs::Current.server = @server
+      @server.activating(&block)
+    end
+
+    def find_server
+      if params[:server_id]
+        MissionControl::Jobs::Current.application.servers[params[:server_id]]
+      else
+        @application.servers.first
+      end
+    end
+end

data/app/controllers/concerns/mission_control/jobs/basic_authentication.rb

@@ -0,0 +1,33 @@
+module MissionControl::Jobs::BasicAuthentication
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :authenticate_by_http_basic
+  end
+
+  private
+    def authenticate_by_http_basic
+      if http_basic_authentication_enabled?
+        if http_basic_authentication_configured?
+          http_basic_authenticate_or_request_with(**http_basic_authentication_credentials)
+        else
+          head :unauthorized
+        end
+      end
+    end
+
+    def http_basic_authentication_enabled?
+      MissionControl::Jobs.http_basic_auth_enabled
+    end
+
+    def http_basic_authentication_configured?
+      http_basic_authentication_credentials.values.all?(&:present?)
+    end
+
+    def http_basic_authentication_credentials
+      {
+        name: MissionControl::Jobs.http_basic_auth_user,
+        password: MissionControl::Jobs.http_basic_auth_password
+      }.transform_values(&:presence)
+    end
+end

data/app/controllers/concerns/mission_control/jobs/failed_jobs_bulk_operations.rb

@@ -0,0 +1,17 @@
+module MissionControl::Jobs::FailedJobsBulkOperations
+  extend ActiveSupport::Concern
+
+  MAX_NUMBER_OF_JOBS_FOR_BULK_OPERATIONS = 3000
+
+  included do
+    include MissionControl::Jobs::JobFilters
+  end
+
+  private
+    # We set a hard limit to prevent problems with the data store (for example, overloading Redis
+    # or causing replication lag in MySQL). This should be enough for most scenarios. For
+    # cases where we need to retry a huge sets of jobs, we offer a runbook that uses the API.
+    def bulk_limited_filtered_failed_jobs
+      ActiveJob.jobs.failed.where(**@job_filters).limit(MAX_NUMBER_OF_JOBS_FOR_BULK_OPERATIONS)
+    end
+end

data/app/controllers/concerns/mission_control/jobs/job_filters.rb

@@ -0,0 +1,41 @@
+module MissionControl::Jobs::JobFilters
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :set_filters
+
+    helper_method :active_filters?, :jobs_filter_param
+  end
+
+  private
+    def set_filters
+      @job_filters = {
+        job_class_name: params.dig(:filter, :job_class_name).to_s.strip.presence,
+        queue_name: params.dig(:filter, :queue_name).to_s.strip.presence,
+        finished_at: finished_at_range_params
+      }.compact
+    end
+
+    def active_filters?
+      @job_filters.any?
+    end
+
+    def jobs_filter_param
+      if @job_filters&.any?
+        { filter: @job_filters }
+      else
+        {}
+      end
+    end
+
+    def finished_at_range_params
+      range_start, range_end = params.dig(:filter, :finished_at_start), params.dig(:filter, :finished_at_end)
+      if range_start || range_end
+        (parse_with_time_zone(range_start)..parse_with_time_zone(range_end))
+      end
+    end
+
+    def parse_with_time_zone(date)
+      Time.zone.parse(date) if date.present?
+    end
+end

data/app/controllers/concerns/mission_control/jobs/job_scoped.rb

@@ -0,0 +1,16 @@
+module MissionControl::Jobs::JobScoped
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :set_job
+  end
+
+  private
+    def set_job
+      @job = jobs_relation.find_by_id!(params[:job_id] || params[:id])
+    end
+
+    def jobs_relation
+      raise NotImplementedError
+    end
+end

data/app/controllers/concerns/mission_control/jobs/not_found_redirections.rb

@@ -0,0 +1,36 @@
+module MissionControl::Jobs::NotFoundRedirections
+  extend ActiveSupport::Concern
+
+  included do
+    rescue_from(ActiveJob::Errors::JobNotFoundError) do |error|
+      redirect_to best_location_for_job_relation(error.job_relation), alert: error.message
+    end
+
+    rescue_from(MissionControl::Jobs::Errors::ResourceNotFound) do |error|
+      redirect_to best_location_for_resource_not_found_error(error), alert: error.message
+    end
+  end
+
+  private
+    def best_location_for_job_relation(job_relation)
+      case
+      when job_relation.failed?
+        application_jobs_path(@application, :failed)
+      when job_relation.queue_name.present?
+        application_queue_path(@application, job_relation.queue_name)
+      else
+        root_path
+      end
+    end
+
+    def best_location_for_resource_not_found_error(error)
+      case error.message
+      when /recurring task/i
+        application_recurring_tasks_path(@application)
+      when /queue/i
+        application_queues_path(@application)
+      else
+        root_url
+      end
+    end
+end

data/app/controllers/concerns/mission_control/jobs/queue_scoped.rb

@@ -0,0 +1,12 @@
+module MissionControl::Jobs::QueueScoped
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :set_queue
+  end
+
+  private
+    def set_queue
+      @queue = ActiveJob.queues[params[:queue_id]] or raise MissionControl::Jobs::Errors::ResourceNotFound, "Queue '#{params[:queue_id]}' not found"
+    end
+end

data/app/controllers/mission_control/jobs/application_controller.rb

@@ -0,0 +1,32 @@
+class MissionControl::Jobs::ApplicationController < MissionControl::Jobs.base_controller_class.constantize
+  ActionController::Base::MODULES.each do |mod|
+    include mod unless self < mod
+  end
+
+  layout "mission_control/jobs/application"
+
+  # Include helpers if not already included
+  helper MissionControl::Jobs::ApplicationHelper unless self < MissionControl::Jobs::ApplicationHelper
+  helper Importmap::ImportmapTagsHelper unless self < Importmap::ImportmapTagsHelper
+
+  include MissionControl::Jobs::BasicAuthentication
+  include MissionControl::Jobs::ApplicationScoped, MissionControl::Jobs::NotFoundRedirections
+  include MissionControl::Jobs::AdapterFeatures
+  include MissionControl::Jobs::JobFilters
+
+  around_action :set_current_locale
+
+  private
+    def default_url_options
+      { server_id: MissionControl::Jobs::Current.server }
+    end
+
+    def set_current_locale(&block)
+      @previous_config = I18n.config
+      I18n.config = MissionControl::Jobs::I18nConfig.new
+      I18n.with_locale(:en, &block)
+    ensure
+      I18n.config = @previous_config
+      @previous_config = nil
+    end
+end

data/app/controllers/mission_control/jobs/bulk_discards_controller.rb

@@ -0,0 +1,20 @@
+class MissionControl::Jobs::BulkDiscardsController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::FailedJobsBulkOperations
+
+  def create
+    jobs_to_discard_count = jobs_to_discard.count
+    jobs_to_discard.discard_all
+
+    redirect_to application_jobs_url(@application, :failed), notice: "Discarded #{jobs_to_discard_count} jobs"
+  end
+
+  private
+    def jobs_to_discard
+      if active_filters?
+        bulk_limited_filtered_failed_jobs
+      else
+        # we don't want to apply any limit since "discarding all" without parameters can be optimized in the adapter as a much faster operation
+        ActiveJob.jobs.failed
+      end
+    end
+end

data/app/controllers/mission_control/jobs/bulk_retries_controller.rb

@@ -0,0 +1,10 @@
+class MissionControl::Jobs::BulkRetriesController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::FailedJobsBulkOperations
+
+  def create
+    jobs_to_retry_count = bulk_limited_filtered_failed_jobs.count
+    bulk_limited_filtered_failed_jobs.retry_all
+
+    redirect_to application_jobs_url(@application, :failed), notice: "Retried #{jobs_to_retry_count} jobs"
+  end
+end

data/app/controllers/mission_control/jobs/discards_controller.rb

@@ -0,0 +1,18 @@
+class MissionControl::Jobs::DiscardsController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::JobScoped
+
+  def create
+    @job.discard
+    redirect_to redirect_location, notice: "Discarded job with id #{@job.job_id}"
+  end
+
+  private
+    def jobs_relation
+      ActiveJob.jobs
+    end
+
+    def redirect_location
+      status = @job.status.presence_in(supported_job_statuses) || :failed
+      application_jobs_url(@application, status, **jobs_filter_param)
+    end
+end

data/app/controllers/mission_control/jobs/dispatches_controller.rb

@@ -0,0 +1,18 @@
+class MissionControl::Jobs::DispatchesController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::JobScoped
+
+  def create
+    @job.dispatch
+    redirect_to redirect_location, notice: "Dispatched job with id #{@job.job_id}"
+  end
+
+  private
+    def jobs_relation
+      ActiveJob.jobs
+    end
+
+    def redirect_location
+      status = @job.status.presence_in(supported_job_statuses) || :blocked
+      application_jobs_url(@application, status)
+    end
+end

data/app/controllers/mission_control/jobs/jobs_controller.rb

@@ -0,0 +1,54 @@
+class MissionControl::Jobs::JobsController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::JobScoped
+
+  skip_before_action :set_job, only: :index
+
+  def index
+    @job_class_names = jobs_with_status.job_class_names
+    @queue_names = ActiveJob.queues.map(&:name)
+
+    @jobs_page = MissionControl::Jobs::Page.new(filtered_jobs_with_status, page: params[:page].to_i)
+    @jobs_count = @jobs_page.total_count
+    load_progress_for_page
+  end
+
+  def show
+    load_progress_for_job
+  end
+
+  private
+    def jobs_relation
+      filtered_jobs
+    end
+
+    def filtered_jobs_with_status
+      filtered_jobs.with_status(jobs_status)
+    end
+
+    def jobs_with_status
+      ActiveJob.jobs.with_status(jobs_status)
+    end
+
+    def filtered_jobs
+      ActiveJob.jobs.where(**@job_filters)
+    end
+
+    helper_method :jobs_status
+
+    def jobs_status
+      params[:status].presence&.inquiry
+    end
+
+    def load_progress_for_page
+      return unless jobs_status&.in_progress? && MissionControl::Jobs::Progress.available?
+
+      job_ids = @jobs_page.records.map(&:job_id)
+      @progress_by_job_id = MissionControl::Jobs::Progress.find_multi(job_ids)
+    end
+
+    def load_progress_for_job
+      return unless MissionControl::Jobs::Progress.available?
+
+      @progress = MissionControl::Jobs::Progress.find(@job.job_id)
+    end
+end

data/app/controllers/mission_control/jobs/progresses_controller.rb

@@ -0,0 +1,27 @@
+class MissionControl::Jobs::ProgressesController < MissionControl::Jobs::ApplicationController
+  def show
+    progress = MissionControl::Jobs::Progress.find(params[:job_id]) if MissionControl::Jobs::Progress.available?
+
+    respond_to do |format|
+      format.json { render json: progress_payload(progress) }
+      format.html { render partial: "mission_control/jobs/jobs/progress_bar", locals: { progress: progress, job_id: params[:job_id] } }
+    end
+  end
+
+  private
+    def progress_payload(progress)
+      if progress
+        {
+          job_id: progress.job_id,
+          current: progress.current,
+          total: progress.total,
+          percent: progress.percent,
+          message: progress.message,
+          stop_requested: progress.stop_requested,
+          missing: false
+        }
+      else
+        { job_id: params[:job_id], missing: true }
+      end
+    end
+end

data/app/controllers/mission_control/jobs/queues/pauses_controller.rb

@@ -0,0 +1,15 @@
+class MissionControl::Jobs::Queues::PausesController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::QueueScoped
+
+  def create
+    @queue.pause
+
+    redirect_back fallback_location: application_queues_url(@application)
+  end
+
+  def destroy
+    @queue.resume
+
+    redirect_back fallback_location: application_queues_url(@application)
+  end
+end

data/app/controllers/mission_control/jobs/queues_controller.rb

@@ -0,0 +1,16 @@
+class MissionControl::Jobs::QueuesController < MissionControl::Jobs::ApplicationController
+  before_action :set_queue, only: :show
+
+  def index
+    @queues = ActiveJob.queues.sort_by(&:name)
+  end
+
+  def show
+    @jobs_page = MissionControl::Jobs::Page.new(@queue.jobs, page: params[:page].to_i)
+  end
+
+  private
+    def set_queue
+      @queue = ActiveJob.queues[params[:id]] or raise MissionControl::Jobs::Errors::ResourceNotFound, "Queue '#{params[:id]}' not found"
+    end
+end

data/app/controllers/mission_control/jobs/recurring_tasks_controller.rb

@@ -0,0 +1,38 @@
+class MissionControl::Jobs::RecurringTasksController < MissionControl::Jobs::ApplicationController
+  before_action :ensure_supported_recurring_tasks
+  before_action :set_recurring_task, only: [ :show, :update ]
+  before_action :ensure_recurring_task_can_be_enqueued, only: :update
+
+  def index
+    @recurring_tasks = MissionControl::Jobs::Current.server.recurring_tasks
+  end
+
+  def show
+    @jobs_page = MissionControl::Jobs::Page.new(@recurring_task.jobs, page: params[:page].to_i)
+  end
+
+  def update
+    if (job = @recurring_task.enqueue) && job.successfully_enqueued?
+      redirect_to application_job_path(@application, job.job_id), notice: "Enqueued recurring task #{@recurring_task.id}"
+    else
+      redirect_to application_recurring_task_path(@application, @recurring_task.id), alert: "Something went wrong enqueuing this recurring task"
+    end
+  end
+
+  private
+    def ensure_supported_recurring_tasks
+      unless recurring_tasks_supported?
+        redirect_to root_url, alert: "This server doesn't support recurring tasks"
+      end
+    end
+
+    def set_recurring_task
+      @recurring_task = MissionControl::Jobs::Current.server.find_recurring_task(params[:id])
+    end
+
+    def ensure_recurring_task_can_be_enqueued
+      unless @recurring_task.runnable?
+        redirect_to application_recurring_task_path(@application, @recurring_task.id), alert: "This task can't be enqueued"
+      end
+    end
+end

data/app/controllers/mission_control/jobs/retries_controller.rb

@@ -0,0 +1,13 @@
+class MissionControl::Jobs::RetriesController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::JobScoped
+
+  def create
+    @job.retry
+    redirect_to application_jobs_url(@application, :failed, **jobs_filter_param), notice: "Retried job with id #{@job.job_id}"
+  end
+
+  private
+    def jobs_relation
+      ActiveJob.jobs.failed
+    end
+end

data/app/controllers/mission_control/jobs/stops_controller.rb

@@ -0,0 +1,22 @@
+class MissionControl::Jobs::StopsController < MissionControl::Jobs::ApplicationController
+  include MissionControl::Jobs::JobScoped
+
+  def create
+    unless MissionControl::Jobs::Progress.available?
+      redirect_to redirect_location, alert: "Progress tracking is not installed. Run rails db:migrate to enable cooperative stop."
+      return
+    end
+
+    MissionControl::Jobs::Progress.request_stop(@job.job_id)
+    redirect_to redirect_location, notice: "Stop requested for job #{@job.job_id}. It will stop the next time it reports progress or calls check_stopped!."
+  end
+
+  private
+    def jobs_relation
+      ActiveJob.jobs
+    end
+
+    def redirect_location
+      application_jobs_url(@application, :in_progress, **jobs_filter_param)
+    end
+end

data/app/controllers/mission_control/jobs/workers_controller.rb

@@ -0,0 +1,23 @@
+class MissionControl::Jobs::WorkersController < MissionControl::Jobs::ApplicationController
+  before_action :ensure_exposed_workers
+
+  def index
+    @workers_page = MissionControl::Jobs::Page.new(workers_relation, page: params[:page].to_i)
+    @workers_count = @workers_page.total_count
+  end
+
+  def show
+    @worker = MissionControl::Jobs::Current.server.find_worker(params[:id])
+  end
+
+  private
+    def ensure_exposed_workers
+      unless workers_exposed?
+        redirect_to root_url, alert: "This server doesn't expose workers"
+      end
+    end
+
+    def workers_relation
+      MissionControl::Jobs::Current.server.workers_relation
+    end
+end

data/app/helpers/mission_control/jobs/application_helper.rb

@@ -0,0 +1,9 @@
+module MissionControl::Jobs
+  module ApplicationHelper
+    # Explicit helper inclusion because ApplicationController inherits from the host app.
+    #
+    # We can't rely on +config.action_controller.include_all_helpers = true+ in the host app.
+    include DatesHelper, JobsHelper, NavigationHelper, InterfaceHelper
+    include MissionControl::Jobs::Engine.routes.url_helpers
+  end
+end

data/app/helpers/mission_control/jobs/dates_helper.rb

@@ -0,0 +1,19 @@
+module MissionControl::Jobs::DatesHelper
+  def time_distance_in_words_with_title(time)
+    tag.span time_ago_in_words_with_default_options(time), title: "Since #{time.to_fs(:long)}"
+  end
+
+  def bidirectional_time_distance_in_words_with_title(time)
+    time_distance = if time.past?
+      "#{time_ago_in_words_with_default_options(time)} ago"
+    else
+      "in #{time_ago_in_words_with_default_options(time)}"
+    end
+
+    tag.span time_distance, title: time.to_fs(:long)
+  end
+
+  def time_ago_in_words_with_default_options(time)
+    time_ago_in_words(time, include_seconds: true, locale: :en)
+  end
+end

data/app/helpers/mission_control/jobs/interface_helper.rb

@@ -0,0 +1,23 @@
+module MissionControl::Jobs::InterfaceHelper
+  def blank_status_notice(message)
+    tag.div message, class: "mt-6 has-text-centered is-size-3 has-text-grey"
+  end
+
+  def blank_status_emoji(status)
+    case status.to_s
+    when "failed", "blocked" then "😌"
+    else ""
+    end
+  end
+
+  def modifier_for_status(status)
+    case status.to_s
+    when "failed"      then "is-danger"
+    when "blocked"     then "is-warning"
+    when "finished"    then "is-success"
+    when "scheduled"   then "is-info"
+    when "in_progress" then "is-primary"
+    else "is-primary is-light"
+    end
+  end
+end