solid_job_monitor 1.0.0

data/lib/active_job/queue_adapters/solid_queue_ext.rb

@@ -0,0 +1,313 @@
+module ActiveJob::QueueAdapters::SolidQueueExt
+  include MissionControl::Jobs::Adapter
+  include RecurringTasks, Workers
+
+  def queues
+    queues = SolidQueue::Queue.all
+    queue_names = queues.map(&:name)
+    pauses = SolidQueue::Pause.where(queue_name: queue_names).index_by(&:queue_name)
+    sizes = SolidQueue::ReadyExecution.where(queue_name: queue_names).group(:queue_name).count
+
+    queues.collect do |queue|
+      {
+        name: queue.name,
+        size: sizes[queue.name] || 0,
+        active: pauses[queue.name].nil?
+      }
+    end
+  end
+
+  def queue_size(queue_name)
+    find_queue_by_name(queue_name).size
+  end
+
+  def clear_queue(queue_name)
+    find_queue_by_name(queue_name).clear
+  end
+
+  def pause_queue(queue_name)
+    find_queue_by_name(queue_name).pause
+  end
+
+  def resume_queue(queue_name)
+    find_queue_by_name(queue_name).resume
+  end
+
+  def queue_paused?(queue_name)
+    find_queue_by_name(queue_name).paused?
+  end
+
+  def supported_job_statuses
+    SolidQueueJobs::STATUS_MAP.keys
+  end
+
+  def supported_job_filters(*)
+    [ :queue_name, :job_class_name, :finished_at ]
+  end
+
+  def jobs_count(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).count
+  end
+
+  def fetch_jobs(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).jobs.map do |job|
+      deserialize_and_proxy_solid_queue_job(job, jobs_relation.status)
+    end
+  end
+
+  def retry_all_jobs(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).retry_all
+  end
+
+  def retry_job(job, jobs_relation)
+    find_solid_queue_job!(job.job_id, jobs_relation).retry
+  end
+
+  def discard_all_jobs(jobs_relation)
+    SolidQueueJobs.new(jobs_relation).discard_all
+  end
+
+  def discard_job(job, jobs_relation)
+    find_solid_queue_job!(job.job_id, jobs_relation).discard
+  end
+
+  def dispatch_job(job, jobs_relation)
+    dispatch_immediately find_solid_queue_job!(job.job_id, jobs_relation)
+  end
+
+  def find_job(job_id, *)
+    if job = SolidQueue::Job.where(active_job_id: job_id).order(:id).last
+      deserialize_and_proxy_solid_queue_job job
+    end
+  end
+
+  private
+    def find_queue_by_name(queue_name)
+      SolidQueue::Queue.find_by_name(queue_name)
+    end
+
+    def find_solid_queue_job!(job_id, jobs_relation)
+      find_solid_queue_job(job_id, jobs_relation) or raise ActiveJob::Errors::JobNotFoundError.new(job_id, jobs_relation)
+    end
+
+    def find_solid_queue_job(job_id, jobs_relation)
+      SolidQueueJobs.new(jobs_relation).find_job(job_id)
+    end
+
+    def deserialize_and_proxy_solid_queue_job(solid_queue_job, job_status = nil)
+      job_status ||= status_from_solid_queue_job(solid_queue_job)
+
+      ActiveJob::JobProxy.new(solid_queue_job.arguments).tap do |job|
+        job.status = job_status
+        job.last_execution_error = execution_error_from_solid_queue_job(solid_queue_job) if job_status == :failed
+        job.raw_data = solid_queue_job.as_json
+        job.filtered_raw_data = filter_raw_data_arguments(job.raw_data)
+        job.failed_at = solid_queue_job&.failed_execution&.created_at if job_status == :failed
+        job.finished_at = solid_queue_job.finished_at
+        job.blocked_by = solid_queue_job.concurrency_key
+        job.blocked_until = solid_queue_job&.blocked_execution&.expires_at if job_status == :blocked
+        job.worker_id = solid_queue_job&.claimed_execution&.process_id if job_status == :in_progress
+        job.started_at = solid_queue_job&.claimed_execution&.created_at if job_status == :in_progress
+        job.scheduled_at = solid_queue_job.scheduled_at
+      end
+    end
+
+    def filter_raw_data_arguments(raw_data)
+      raw_data.deep_dup.tap do |filtered_raw_data|
+        filtered_raw_data["arguments"]["arguments"] = MissionControl::Jobs.job_arguments_filter.apply_to(filtered_raw_data.dig("arguments", "arguments"))
+      end
+    end
+
+    def status_from_solid_queue_job(solid_queue_job)
+      SolidQueueJobs::STATUS_MAP.invert[solid_queue_job.status]
+    end
+
+    def execution_error_from_solid_queue_job(solid_queue_job)
+      if solid_queue_job.failed?
+        ActiveJob::ExecutionError.new \
+          error_class: solid_queue_job.failed_execution.exception_class,
+          message: solid_queue_job.failed_execution.message,
+          backtrace: solid_queue_job.failed_execution.backtrace || []
+      end
+    rescue JSON::ParserError
+      ActiveJob::ExecutionError.new \
+        error_class: "",
+        message: solid_queue_job.failed_execution.error_before_type_cast,
+        backtrace: []
+    end
+
+    def dispatch_immediately(job)
+      if job.blocked?
+        SolidQueue::Job.transaction do
+          job.dispatch_bypassing_concurrency_limits
+          job.blocked_execution.destroy!
+        end
+      else
+        job.scheduled_execution.update!(scheduled_at: Time.now)
+      end
+    end
+
+    class SolidQueueJobs
+      STATUS_MAP = {
+        pending: :ready,
+        failed: :failed,
+        in_progress: :claimed,
+        blocked: :blocked,
+        scheduled: :scheduled,
+        finished: :finished
+      }
+
+      def initialize(jobs_relation)
+        @jobs_relation = jobs_relation
+      end
+
+      def jobs
+        solid_queue_status.finished? ? order_finished_jobs(finished_jobs) : order_executions(executions).map(&:job).compact
+      end
+
+      def count
+        limit_value_provided? ? direct_count : internally_limited_count
+      end
+
+      def find_job(active_job_id)
+        if job = SolidQueue::Job.where(active_job_id: active_job_id).order(:id).last
+          job if matches_relation_filters?(job)
+        end
+      end
+
+      def discard_all
+        execution_class_by_status.discard_all_from_jobs(jobs)
+      end
+
+      def retry_all
+        SolidQueue::FailedExecution.retry_all(jobs)
+      end
+
+      private
+        attr_reader :jobs_relation
+
+        delegate :queue_name, :limit_value, :limit_value_provided?, :offset_value, :job_class_name,
+          :default_page_size, :worker_id, :recurring_task_id, :finished_at, to: :jobs_relation
+
+        def executions
+          execution_class_by_status
+            .then { |executions| include_execution_association(executions) }
+            .then { |executions| filter_executions_by_queue(executions) }
+            .then { |executions| filter_executions_by_class(executions) }
+            .then { |executions| filter_executions_by_process_id(executions) }
+            .then { |executions| filter_executions_by_task_key(executions) }
+            .then { |executions| limit(executions) }
+            .then { |executions| offset(executions) }
+        end
+
+        def finished_jobs
+          SolidQueue::Job.finished
+            .then { |jobs| filter_jobs_by_queue(jobs) }
+            .then { |jobs| filter_jobs_by_class(jobs) }
+            .then { |jobs| filter_jobs_by_finished_at(jobs) }
+            .then { |jobs| limit(jobs) }
+            .then { |jobs| offset(jobs) }
+        end
+
+        def order_finished_jobs(jobs)
+          jobs.order(finished_at: :desc)
+        end
+
+        def order_executions(executions)
+          case
+            # Follow polling order for scheduled executions, the rest by job_id, desc or asc
+          when solid_queue_status.scheduled? then executions.ordered
+          when recurring_task_id.present?    then executions.order(job_id: :desc)
+          else executions.order(job_id: :asc)
+          end
+        end
+
+        def matches_relation_filters?(job)
+          matches_status?(job) && matches_queue_name?(job)
+        end
+
+        def direct_count
+          solid_queue_status.finished? ? finished_jobs.count : executions.count
+        end
+
+        def internally_limited_count
+          count_limit = MissionControl::Jobs.internal_query_count_limit + 1
+          limited_count = solid_queue_status.finished? ? finished_jobs.limit(count_limit).count : executions.limit(count_limit).count
+          (limited_count == count_limit) ? Float::INFINITY : limited_count
+        end
+
+        def execution_class_by_status
+          if recurring_task_id.present?
+            SolidQueue::RecurringExecution
+          elsif solid_queue_status.present? && !solid_queue_status.finished?
+            "SolidQueue::#{solid_queue_status.capitalize}Execution".safe_constantize
+          else
+            raise ActiveJob::Errors::QueryError, "Status not supported: #{jobs_relation.status}"
+          end
+        end
+
+        def include_execution_association(executions)
+          solid_queue_status.present? ? executions.includes(job: "#{solid_queue_status}_execution") : executions.includes(:job)
+        end
+
+        def filter_executions_by_queue(executions)
+          return executions unless queue_name.present?
+
+          if solid_queue_status.ready?
+            executions.where(queue_name: queue_name)
+          else
+            executions.where(job: { queue_name: queue_name })
+          end
+        end
+
+        def filter_jobs_by_queue(jobs)
+          queue_name.present? ? jobs.where(queue_name: queue_name) : jobs
+        end
+
+        def filter_executions_by_class(executions)
+          job_class_name.present? ? executions.where(job: { class_name: job_class_name }) : executions
+        end
+
+        def filter_executions_by_process_id(executions)
+          return executions unless worker_id.present?
+
+          if solid_queue_status.claimed?
+            executions.where(process_id: worker_id)
+          else
+            raise ActiveJob::Errors::QueryError, "Filtering by worker id is not supported for status #{jobs_relation.status}"
+          end
+        end
+
+        def filter_executions_by_task_key(executions)
+          recurring_task_id.present? ? executions.where(task_key: recurring_task_id) : executions
+        end
+
+        def filter_jobs_by_class(jobs)
+          job_class_name.present? ? jobs.where(class_name: job_class_name) : jobs
+        end
+
+        def filter_jobs_by_finished_at(jobs)
+          finished_at.present? ? jobs.where(finished_at: finished_at) : jobs
+        end
+
+        def limit(executions_or_jobs)
+          limit_value.present? ? executions_or_jobs.limit(limit_value) : executions_or_jobs
+        end
+
+        def offset(executions_or_jobs)
+          offset_value.present? ? executions_or_jobs.offset(offset_value) : executions_or_jobs
+        end
+
+        def matches_status?(job)
+          solid_queue_status.blank? || job.public_send("#{solid_queue_status}?")
+        end
+
+        def matches_queue_name?(job)
+          queue_name.blank? || job.queue_name == queue_name
+        end
+
+        def solid_queue_status
+          STATUS_MAP[jobs_relation.status].to_s.inquiry
+        end
+    end
+end

data/lib/active_job/queues.rb

@@ -0,0 +1,33 @@
+# An enumerable collection of queues that supports direct access to queues by name.
+#
+#   queue_1 = ApplicationJob::Queue.new("queue_1")
+#   queue_2 = ApplicationJob::Queue.new("queue_2")
+#   queues = ApplicationJob::Queues.new([queue_1, queue_2])
+#
+#   queues[:queue_1] #=> queue_1
+#   queues[:queue_2] #=> queue_2
+#   queues.to_a #=> [ queue_1, queue_2 ] # Enumerable
+#
+# See +ActiveJob::Queue+.
+class ActiveJob::Queues
+  include Enumerable
+
+  delegate :each, to: :values
+  delegate :values, to: :queues_by_id, private: true
+  delegate :size, :length, :to_s, :inspect, to: :queues_by_id
+
+  def initialize(queues)
+    @queues_by_id = queues.index_by(&:id).with_indifferent_access
+  end
+
+  def to_h
+    queues_by_id.dup
+  end
+
+  def [](name)
+    queues_by_id[name.to_s.parameterize]
+  end
+
+  private
+    attr_reader :queues_by_id
+end

data/lib/mission_control/jobs/adapter.rb

@@ -0,0 +1,156 @@
+module MissionControl::Jobs::Adapter
+  def activating(&block)
+    block.call
+  end
+
+  def supports_job_status?(status)
+    supported_job_statuses.include?(status)
+  end
+
+  def supported_job_statuses
+    # All adapters need to support these at a minimum
+    [ :pending, :failed ]
+  end
+
+  def supports_job_filter?(jobs_relation, filter)
+    supported_job_filters(jobs_relation).include?(filter)
+  end
+
+  # List of filters supported natively. Non-supported filters are done in memory.
+  def supported_job_filters(jobs_relation)
+    []
+  end
+
+  def supports_queue_pausing?
+    true
+  end
+
+  def exposes_workers?
+    false
+  end
+
+  def supports_recurring_tasks?
+    false
+  end
+
+  # Returns an array with the list of recurring tasks. Each task is represented as a hash
+  # with these attributes:
+  #   {
+  #     id: "periodic-job",
+  #     job_class_name: "MyJob",
+  #     arguments: [ 123, { arg: :value }]
+  #     schedule: "every monday at 9 am",
+  #     last_enqueued_at: Fri, 26 Jan 2024 20:31:09.652174000 UTC +00:00,
+  #   }
+  def recurring_tasks
+    if supports_recurring_tasks?
+      raise_incompatible_adapter_error_from :recurring_tasks
+    end
+  end
+
+  # Returns a recurring task represented by a hash as indicated above
+  def find_recurring_task(recurring_task_id)
+    if supports_recurring_tasks?
+      raise_incompatible_adapter_error_from :find_recurring_task
+    end
+  end
+
+
+  # Returns an array with the list of workers. Each worker is represented as a hash
+  # with these attributes:
+  #   {
+  #     id: 123,
+  #     name: "worker-name",
+  #     hostname: "hey-default-101",
+  #     last_heartbeat_at: Fri, 26 Jan 2024 20:31:09.652174000 UTC +00:00,
+  #     configuration: { ... }
+  #     raw_data: { ... }
+  #   }
+  def workers
+    if exposes_workers?
+      raise_incompatible_adapter_error_from :workers
+    end
+  end
+
+  # Returns a worker represented by a hash as indicated above
+  def find_worker(worker_id)
+    if exposes_workers?
+      raise_incompatible_adapter_error_from :find_worker
+    end
+  end
+
+
+  # Returns an array with the list of queues. Each queue is represented as a hash
+  # with these attributes:
+  #   {
+  #     name: "queue_name",
+  #     size: 1,
+  #     active: true
+  #   }
+  def queues
+    raise_incompatible_adapter_error_from :queue_names
+  end
+
+  def queue_size(queue_name)
+    raise_incompatible_adapter_error_from :queue_size
+  end
+
+  def clear_queue(queue_name)
+    raise_incompatible_adapter_error_from :clear_queue
+  end
+
+  def pause_queue(queue_name)
+    if supports_queue_pausing?
+      raise_incompatible_adapter_error_from :pause_queue
+    end
+  end
+
+  def resume_queue(queue_name)
+    if supports_queue_pausing?
+      raise_incompatible_adapter_error_from :resume_queue
+    end
+  end
+
+  def queue_paused?(queue_name)
+    if supports_queue_pausing?
+      raise_incompatible_adapter_error_from :queue_paused?
+    end
+  end
+
+  def jobs_count(jobs_relation)
+    raise_incompatible_adapter_error_from :jobs_count
+  end
+
+  def fetch_jobs(jobs_relation)
+    raise_incompatible_adapter_error_from :fetch_jobs
+  end
+
+  def retry_all_jobs(jobs_relation)
+    raise_incompatible_adapter_error_from :retry_all_jobs
+  end
+
+  def retry_job(job, jobs_relation)
+    raise_incompatible_adapter_error_from :retry_job
+  end
+
+  def discard_all_jobs(jobs_relation)
+    raise_incompatible_adapter_error_from :discard_all_jobs
+  end
+
+  def discard_job(job, jobs_relation)
+    raise_incompatible_adapter_error_from :discard_job
+  end
+
+  def dispatch_job(job, jobs_relation)
+    raise_incompatible_adapter_error_from :dispatch_job
+  end
+
+  def find_job(job_id, *)
+    raise_incompatible_adapter_error_from :find_job
+  end
+
+  private
+    def raise_incompatible_adapter_error_from(method_name)
+      raise MissionControl::Jobs::Errors::IncompatibleAdapter, "Adapter #{ActiveJob.adapter_name(self)} must implement `#{method_name}`"
+    end
+end

data/lib/mission_control/jobs/application.rb

@@ -0,0 +1,20 @@
+# An application containing backend jobs servers
+class MissionControl::Jobs::Application
+  include MissionControl::Jobs::IdentifiedByName
+
+  attr_reader :servers
+
+  def initialize(name:)
+    super
+    @servers = MissionControl::Jobs::IdentifiedElements.new
+  end
+
+  def add_servers(queue_adapters_by_name)
+    queue_adapters_by_name.each do |name, queue_adapter|
+      adapter, cleaner = queue_adapter
+
+      servers << MissionControl::Jobs::Server.new(name: name.to_s, queue_adapter: adapter,
+        backtrace_cleaner: cleaner, application: self)
+    end
+  end
+end

data/lib/mission_control/jobs/applications.rb

@@ -0,0 +1,8 @@
+# A container to register applications
+class MissionControl::Jobs::Applications < MissionControl::Jobs::IdentifiedElements
+  def add(name, queue_adapters_by_name = {})
+    self << MissionControl::Jobs::Application.new(name: name).tap do |application|
+      application.add_servers(queue_adapters_by_name)
+    end
+  end
+end

data/lib/mission_control/jobs/arguments_filter.rb

@@ -0,0 +1,24 @@
+# Replaces argument values with [FILTERED] for any keys that match a filter.
+class MissionControl::Jobs::ArgumentsFilter
+  FILTERED = "[FILTERED]"
+
+  def initialize(filter)
+    @filter = filter
+  end
+
+  def apply_to(arguments)
+    case arguments
+    when Array
+      arguments.map { |a| apply_to(a) }
+    when Hash
+      arguments.map do |k, v|
+        [ k, filter.include?(k.to_s) ? FILTERED : v ]
+      end.to_h
+    else
+      arguments
+    end
+  end
+
+  private
+    attr_reader :filter
+end

data/lib/mission_control/jobs/authentication.rb

@@ -0,0 +1,67 @@
+require "rails/command"
+
+class MissionControl::Jobs::Authentication < Rails::Command::Base
+  def self.configure
+    new.configure
+  end
+
+  def configure
+    if credentials_accessible?
+      if authentication_configured?
+        say "HTTP Basic Authentication is already configured for `#{Rails.env}`. You can edit it using `credentials:edit`"
+      else
+        say "Setting up credentials for HTTP Basic Authentication for `#{Rails.env}` environment."
+        say ""
+
+        username = ask "Enter username: "
+        password = SecureRandom.base58(64)
+
+        store_credentials(username, password)
+        say "Username and password stored in Rails encrypted credentials."
+        say ""
+        say "You can now access Mission Control – Jobs with: "
+        say ""
+        say " - Username: #{username}"
+        say " - password: #{password}"
+        say ""
+        say "You can also edit these in the future via `credentials:edit`"
+      end
+    else
+      say "Rails credentials haven't been configured or aren't accessible. Configure them following the instructions in `credentials:help`"
+    end
+  end
+
+  private
+    attr_reader :environment
+
+    def credentials_accessible?
+      credentials.read.present?
+    end
+
+    def authentication_configured?
+      %i[ http_basic_auth_user http_basic_auth_password ].any? do |key|
+        credentials.dig(:mission_control, key).present?
+      end
+    end
+
+    def store_credentials(username, password)
+      content = credentials.read + "\n" + http_authentication_entry(username, password) + "\n"
+      credentials.write(content)
+    end
+
+    def credentials
+      @credentials ||= Rails.application.encrypted(config.content_path, key_path: config.key_path)
+    end
+
+    def config
+      Rails.application.config.credentials
+    end
+
+    def http_authentication_entry(username, password)
+      <<~ENTRY
+        mission_control:
+          http_basic_auth_user: #{username}
+          http_basic_auth_password: #{password}
+      ENTRY
+    end
+end

data/lib/mission_control/jobs/console/connect_to.rb

@@ -0,0 +1,15 @@
+require "irb/command"
+
+module MissionControl::Jobs::Console
+  class ConnectTo < IRB::Command::Base
+    category "Mission control jobs"
+    description "Connect to a job server"
+
+    def execute(server_locator)
+      server = MissionControl::Jobs::Server.from_global_id(server_locator)
+      MissionControl::Jobs::Current.server = server
+
+      puts "Connected to #{server_locator}"
+    end
+  end
+end

data/lib/mission_control/jobs/console/context.rb

@@ -0,0 +1,11 @@
+module MissionControl::Jobs::Console::Context
+  mattr_accessor :jobs_server
+
+  def evaluate(*)
+    if MissionControl::Jobs::Current.server
+      MissionControl::Jobs::Current.server.activating { super }
+    else
+      super
+    end
+  end
+end

data/lib/mission_control/jobs/console/jobs_help.rb

@@ -0,0 +1,23 @@
+require "irb/command"
+
+module MissionControl::Jobs::Console
+  class JobsHelp < IRB::Command::Base
+    category "Mission control jobs"
+    description "Show help for managing jobs"
+
+    def execute(*)
+      puts "You are currently connected to #{MissionControl::Jobs::Current.server}" if MissionControl::Jobs::Current.server
+
+      puts "You can connect to a job server with"
+      puts %(  connect_to <app_id>:<server_id>\n\n)
+
+      puts "Available job servers:\n"
+
+      MissionControl::Jobs.applications.each do |application|
+        application.servers.each do |server|
+          puts "  * #{server.to_global_id}"
+        end
+      end
+    end
+  end
+end