mission_control-jobs 0.1.0

data/lib/active_job/queue_adapters/solid_queue_ext.rb

@@ -0,0 +1,294 @@
+module ActiveJob::QueueAdapters::SolidQueueExt
+  include MissionControl::Jobs::Adapter
+
+  def queues
+    queues = SolidQueue::Queue.all
+    pauses = SolidQueue::Pause.where(queue_name: queues.map(&:name)).index_by(&:queue_name)
+
+    queues.collect do |queue|
+      {
+        name: queue.name,
+        size: queue.size,
+        active: pauses[queue.name].nil?
+      }
+    end
+  end
+
+  def queue_size(queue_name)
+    find_queue_by_name(queue_name).size
+  end
+
+  def clear_queue(queue_name)
+    find_queue_by_name(queue_name).clear
+  end
+
+  def pause_queue(queue_name)
+    find_queue_by_name(queue_name).pause
+  end
+
+  def resume_queue(queue_name)
+    find_queue_by_name(queue_name).resume
+  end
+
+  def queue_paused?(queue_name)
+    find_queue_by_name(queue_name).paused?
+  end
+
+  def supported_statuses
+    RelationAdapter::STATUS_MAP.keys
+  end
+
+  def supported_filters(*)
+    [ :queue_name, :job_class_name ]
+  end
+
+  def exposes_workers?
+    true
+  end
+
+  def workers
+    SolidQueue::Process.where(kind: "Worker").collect do |process|
+      worker_attributes_from_solid_queue_process(process)
+    end
+  end
+
+  def find_worker(worker_id)
+    if process = SolidQueue::Process.find_by(id: worker_id)
+      worker_attributes_from_solid_queue_process(process)
+    end
+  end
+
+  def jobs_count(jobs_relation)
+    RelationAdapter.new(jobs_relation).count
+  end
+
+  def fetch_jobs(jobs_relation)
+    find_solid_queue_jobs_within(jobs_relation).map { |job| deserialize_and_proxy_solid_queue_job(job, jobs_relation.status) }
+  end
+
+  def retry_all_jobs(jobs_relation)
+    RelationAdapter.new(jobs_relation).retry_all
+  end
+
+  def retry_job(job, jobs_relation)
+    find_solid_queue_job!(job.job_id, jobs_relation).retry
+  end
+
+  def discard_all_jobs(jobs_relation)
+    RelationAdapter.new(jobs_relation).discard_all
+  end
+
+  def discard_job(job, jobs_relation)
+    find_solid_queue_job!(job.job_id, jobs_relation).discard
+  end
+
+  def find_job(job_id, *)
+    if job = SolidQueue::Job.where(active_job_id: job_id).order(:id).last
+      deserialize_and_proxy_solid_queue_job job
+    end
+  end
+
+  private
+    def find_queue_by_name(queue_name)
+      SolidQueue::Queue.find_by_name(queue_name)
+    end
+
+    def worker_attributes_from_solid_queue_process(process)
+      {
+        id: process.id,
+        name: "PID: #{process.pid}",
+        hostname: process.hostname,
+        last_heartbeat_at: process.last_heartbeat_at,
+        configuration: process.metadata,
+        raw_data: process.as_json
+      }
+    end
+
+    def find_solid_queue_job!(job_id, jobs_relation)
+      find_solid_queue_job(job_id, jobs_relation) or raise ActiveJob::Errors::JobNotFoundError.new(job_id, jobs_relation)
+    end
+
+    def find_solid_queue_job(job_id, jobs_relation)
+      RelationAdapter.new(jobs_relation).find_job(job_id)
+    end
+
+    def find_solid_queue_jobs_within(jobs_relation)
+      RelationAdapter.new(jobs_relation).jobs
+    end
+
+    def deserialize_and_proxy_solid_queue_job(solid_queue_job, job_status = nil)
+      job_status ||= status_from_solid_queue_job(solid_queue_job)
+
+      ActiveJob::JobProxy.new(solid_queue_job.arguments).tap do |job|
+        job.status = job_status
+        job.last_execution_error = execution_error_from_solid_queue_job(solid_queue_job) if job_status == :failed
+        job.raw_data = solid_queue_job.as_json
+        job.failed_at = solid_queue_job&.failed_execution&.created_at if job_status == :failed
+        job.finished_at = solid_queue_job.finished_at
+        job.blocked_by = solid_queue_job.concurrency_key
+        job.blocked_until = solid_queue_job&.blocked_execution&.expires_at if job_status == :blocked
+        job.worker_id = solid_queue_job&.claimed_execution&.process_id if job_status == :in_progress
+        job.started_at = solid_queue_job&.claimed_execution&.created_at if job_status == :in_progress
+        job.scheduled_at = solid_queue_job.scheduled_at
+      end
+    end
+
+    def status_from_solid_queue_job(solid_queue_job)
+      RelationAdapter::STATUS_MAP.invert[solid_queue_job.status]
+    end
+
+    def execution_error_from_solid_queue_job(solid_queue_job)
+      if solid_queue_job.failed?
+        ActiveJob::ExecutionError.new \
+          error_class: solid_queue_job.failed_execution.exception_class,
+          message: solid_queue_job.failed_execution.message,
+          backtrace: solid_queue_job.failed_execution.backtrace
+      end
+    end
+
+    class RelationAdapter
+      STATUS_MAP = {
+        pending: :ready,
+        failed: :failed,
+        in_progress: :claimed,
+        blocked: :blocked,
+        scheduled: :scheduled,
+        finished: :finished
+      }
+
+      def initialize(jobs_relation)
+        @jobs_relation = jobs_relation
+      end
+
+      def jobs
+        solid_queue_status.finished? ? order_finished_jobs(finished_jobs) : order_executions(executions).map(&:job)
+      end
+
+      def count
+        limit_value_provided? ? direct_count : internally_limited_count
+      end
+
+      def find_job(active_job_id)
+        if job = SolidQueue::Job.find_by(active_job_id: active_job_id)
+          job if matches_relation_filters?(job)
+        end
+      end
+
+      def discard_all
+        execution_class_by_status.discard_all_from_jobs(jobs)
+      end
+
+      def retry_all
+        SolidQueue::FailedExecution.retry_all(jobs)
+      end
+
+      private
+        attr_reader :jobs_relation
+
+        delegate :queue_name, :limit_value, :limit_value_provided?, :offset_value, :job_class_name, :default_page_size, :worker_id, to: :jobs_relation
+
+        def executions
+          execution_class_by_status.includes(job: "#{solid_queue_status}_execution")
+            .then { |executions| filter_executions_by_queue(executions) }
+            .then { |executions| filter_executions_by_class(executions) }
+            .then { |executions| filter_executions_by_process_id(executions) }
+            .then { |executions| limit(executions) }
+            .then { |executions| offset(executions) }
+        end
+
+        def finished_jobs
+          SolidQueue::Job.finished
+            .then { |jobs| filter_jobs_by_queue(jobs) }
+            .then { |jobs| filter_jobs_by_class(jobs) }
+            .then { |jobs| limit(jobs) }
+            .then { |jobs| offset(jobs) }
+        end
+
+        def order_finished_jobs(jobs)
+          jobs.order(finished_at: :desc)
+        end
+
+        def order_executions(executions)
+          # Follow polling order for scheduled executions, the rest by job_id
+          if solid_queue_status.scheduled? then executions.ordered
+          else
+            executions.order(:job_id)
+          end
+        end
+
+        def matches_relation_filters?(job)
+          matches_status?(job) && matches_queue_name?(job)
+        end
+
+        def direct_count
+          solid_queue_status.finished? ? finished_jobs.count : executions.count
+        end
+
+        INTERNAL_COUNT_LIMIT = 500_000 # Hard limit to keep unlimited count queries fast enough
+
+        def internally_limited_count
+          limited_count = solid_queue_status.finished? ? finished_jobs.limit(INTERNAL_COUNT_LIMIT + 1).count : executions.limit(INTERNAL_COUNT_LIMIT + 1).count
+          (limited_count == INTERNAL_COUNT_LIMIT + 1) ? Float::INFINITY : limited_count
+        end
+
+        def execution_class_by_status
+          if solid_queue_status.present? && !solid_queue_status.finished?
+            "SolidQueue::#{solid_queue_status.capitalize}Execution".safe_constantize
+          else
+            raise ActiveJob::Errors::QueryError, "Status not supported: #{jobs_relation.status}"
+          end
+        end
+
+        def filter_executions_by_queue(executions)
+          return executions unless queue_name.present?
+
+          if solid_queue_status.ready?
+            executions.where(queue_name: queue_name)
+          else
+            executions.where(job: { queue_name: queue_name })
+          end
+        end
+
+        def filter_jobs_by_queue(jobs)
+          queue_name.present? ? jobs.where(queue_name: queue_name) : jobs
+        end
+
+        def filter_executions_by_class(executions)
+          job_class_name.present? ? executions.where(job: { class_name: job_class_name }) : executions
+        end
+
+        def filter_executions_by_process_id(executions)
+          return executions unless worker_id.present?
+
+          if solid_queue_status.claimed?
+            executions.where(process_id: worker_id)
+          else
+            raise ActiveJob::Errors::QueryError, "Filtering by worker ID is not supported for status #{jobs_relation.status}"
+          end
+        end
+
+        def filter_jobs_by_class(jobs)
+          job_class_name.present? ? jobs.where(class_name: job_class_name) : jobs
+        end
+
+        def limit(executions_or_jobs)
+          limit_value.present? ? executions_or_jobs.limit(limit_value) : executions_or_jobs
+        end
+
+        def offset(executions_or_jobs)
+          offset_value.present? ? executions_or_jobs.offset(offset_value) : executions_or_jobs
+        end
+
+        def matches_status?(job)
+          solid_queue_status.blank? || job.public_send("#{solid_queue_status}?")
+        end
+
+        def matches_queue_name?(job)
+          queue_name.blank? || job.queue_name == queue_name
+        end
+
+        def solid_queue_status
+          STATUS_MAP[jobs_relation.status].to_s.inquiry
+        end
+    end
+end

data/lib/active_job/queues.rb

@@ -0,0 +1,29 @@
+# An enumerable collection of queues that supports direct access to queues by name.
+#
+#   queue_1 = ApplicationJob::Queue.new("queue_1")
+#   queue_2 = ApplicationJob::Queue.new("queue_2")
+#   queues = ApplicationJob::Queues.new([queue_1, queue_2])
+#
+#   queues[:queue_1] #=> queue_1
+#   queues[:queue_2] #=> queue_2
+#   queues.to_a #=> [ queue_1, queue_2 ] # Enumerable
+#
+# See +ActiveJob::Queue+.
+class ActiveJob::Queues
+  include Enumerable
+
+  delegate :each, to: :values
+  delegate :values, to: :queues_by_name, private: true
+  delegate :[], :size, :length, :to_s, :inspect, to: :queues_by_name
+
+  def initialize(queues)
+    @queues_by_name = queues.index_by(&:name).with_indifferent_access
+  end
+
+  def to_h
+    queues_by_name.dup
+  end
+
+  private
+    attr_reader :queues_by_name
+end

data/lib/mission_control/jobs/adapter.rb

@@ -0,0 +1,108 @@
+module MissionControl::Jobs::Adapter
+  def activating(&block)
+    block.call
+  end
+
+  def supported_statuses
+    # All adapters need to support these at a minimum
+    [ :pending, :failed ]
+  end
+
+  def supports_filter?(jobs_relation, filter)
+    supported_filters(jobs_relation).include?(filter)
+  end
+
+  # List of filters supported natively. Non-supported filters are done in memory.
+  def supported_filters(jobs_relation)
+    []
+  end
+
+  def supports_queue_pausing?
+    true
+  end
+
+  def exposes_workers?
+    false
+  end
+
+  # Returns an array with the list of workers. Each worker is represented as a hash
+  # with these attributes:
+  #   {
+  #     id: 123,
+  #     name: "adapter-name",
+  #     hostname: "hey-default-101",
+  #     last_heartbeat_at: Fri, 26 Jan 2024 20:31:09.652174000 UTC +00:00,
+  #     configuration: { ... }
+  #     raw_data: { ... }
+  #   }
+  def workers
+    if exposes_workers?
+      raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `workers`")
+    end
+  end
+
+  # Returns an array with the list of queues. Each queue is represented as a hash
+  # with these attributes:
+  #   {
+  #     name: "queue_name",
+  #     size: 1,
+  #     active: true
+  #   }
+  def queues
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `queue_names`")
+  end
+
+  def queue_size(queue_name)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `queue_size`")
+  end
+
+  def clear_queue(queue_name)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `clear_queue`")
+  end
+
+  def pause_queue(queue_name)
+    if supports_queue_pausing?
+      raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `pause_queue`")
+    end
+  end
+
+  def resume_queue(queue_name)
+    if supports_queue_pausing?
+      raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `resume_queue`")
+    end
+  end
+
+  def queue_paused?(queue_name)
+    if supports_queue_pausing?
+      raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `queue_paused?`")
+    end
+  end
+
+  def jobs_count(jobs_relation)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `jobs_count`")
+  end
+
+  def fetch_jobs(jobs_relation)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `fetch_jobs`")
+  end
+
+  def retry_all_jobs(jobs_relation)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `retry_all_jobs`")
+  end
+
+  def retry_job(job, jobs_relation)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `retry_job`")
+  end
+
+  def discard_all_jobs(jobs_relation)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `discard_all_jobs`")
+  end
+
+  def discard_job(job, jobs_relation)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `discard_job`")
+  end
+
+  def find_job(job_id, *)
+    raise MissionControl::Jobs::Errors::IncompatibleAdapter("Adapter must implement `find_job`")
+  end
+end

data/lib/mission_control/jobs/application.rb

@@ -0,0 +1,17 @@
+# An application containing backend jobs servers
+class MissionControl::Jobs::Application
+  include MissionControl::Jobs::IdentifiedByName
+
+  attr_reader :servers
+
+  def initialize(name:)
+    super
+    @servers = MissionControl::Jobs::IdentifiedElements.new
+  end
+
+  def add_servers(queue_adapters_by_name)
+    queue_adapters_by_name.each do |name, queue_adapter|
+      servers << MissionControl::Jobs::Server.new(name: name.to_s, queue_adapter: queue_adapter, application: self)
+    end
+  end
+end

data/lib/mission_control/jobs/applications.rb

@@ -0,0 +1,8 @@
+# A container to register applications
+class MissionControl::Jobs::Applications < MissionControl::Jobs::IdentifiedElements
+  def add(name, queue_adapters_by_name = {})
+    self << MissionControl::Jobs::Application.new(name: name).tap do |application|
+      application.add_servers(queue_adapters_by_name)
+    end
+  end
+end

data/lib/mission_control/jobs/console/context.rb

@@ -0,0 +1,11 @@
+module MissionControl::Jobs::Console::Context
+  mattr_accessor :jobs_server
+
+  def evaluate(*)
+    if MissionControl::Jobs::Current.server
+      MissionControl::Jobs::Current.server.activating { super }
+    else
+      super
+    end
+  end
+end

data/lib/mission_control/jobs/console/helpers.rb

@@ -0,0 +1,26 @@
+module MissionControl::Jobs::Console::Helpers
+  def connect_to(server_locator)
+    server = MissionControl::Jobs::Server.from_global_id(server_locator)
+    MissionControl::Jobs::Current.server = server
+
+    puts "Connected to #{server_locator}"
+    nil
+  end
+
+  def jobs_help
+    puts "You are currently connected to #{MissionControl::Jobs::Current.server}" if MissionControl::Jobs::Current.server
+
+    puts "You can connect to a job server with"
+    puts "  connect_to <app_id>:<server_id>\n\n"
+
+    puts "Available job servers:\n"
+
+    MissionControl::Jobs.applications.each do |application|
+      application.servers.each do |server|
+        puts "  * #{server.to_global_id}"
+      end
+    end
+
+    nil
+  end
+end

data/lib/mission_control/jobs/engine.rb

@@ -0,0 +1,88 @@
+require "mission_control/jobs/version"
+require "mission_control/jobs/engine"
+
+require "importmap-rails"
+require "turbo-rails"
+require "stimulus-rails"
+
+module MissionControl
+  module Jobs
+    class Engine < ::Rails::Engine
+      isolate_namespace MissionControl::Jobs
+
+      config.mission_control = ActiveSupport::OrderedOptions.new unless config.try(:mission_control)
+      config.mission_control.jobs = ActiveSupport::OrderedOptions.new
+
+      config.before_initialize do
+        config.mission_control.jobs.applications = MissionControl::Jobs::Applications.new
+
+        config.mission_control.jobs.each do |key, value|
+          MissionControl::Jobs.public_send("#{key}=", value)
+        end
+
+        if config.active_job.queue_adapter.present? && MissionControl::Jobs.adapters.empty?
+          MissionControl::Jobs.adapters << config.active_job.queue_adapter
+        end
+      end
+
+      initializer "mission_control-jobs.active_job.extensions" do
+        ActiveSupport.on_load :active_job do
+          include ActiveJob::Querying
+          include ActiveJob::Executing
+          include ActiveJob::Failed
+          ActiveJob.extend ActiveJob::Querying::Root
+        end
+      end
+
+      config.before_initialize do
+        if MissionControl::Jobs.adapters.include?(:resque)
+          ActiveJob::QueueAdapters::ResqueAdapter.prepend ActiveJob::QueueAdapters::ResqueExt
+          Resque.prepend Resque::ThreadSafeRedis
+        end
+
+        if MissionControl::Jobs.adapters.include?(:solid_queue)
+          ActiveJob::QueueAdapters::SolidQueueAdapter.prepend ActiveJob::QueueAdapters::SolidQueueExt
+        end
+      end
+
+      config.after_initialize do |app|
+        unless app.config.eager_load
+          # When loading classes lazily (development), we want to make sure
+          # the base host +ApplicationController+ class is loaded when loading the
+          # Engine's +ApplicationController+, or it will fail to load the class.
+          MissionControl::Jobs.base_controller_class.constantize
+        end
+
+        if MissionControl::Jobs.applications.empty?
+          queue_adapters_by_name = MissionControl::Jobs.adapters.each_with_object({}) do |adapter, hsh|
+            hsh[adapter] = ActiveJob::QueueAdapters.lookup(adapter).new
+          end
+
+          MissionControl::Jobs.applications.add(app.class.module_parent.name, queue_adapters_by_name)
+        end
+      end
+
+      console do
+        require "irb/context"
+
+        IRB::Context.prepend(MissionControl::Jobs::Console::Context)
+        Rails::ConsoleMethods.include(MissionControl::Jobs::Console::Helpers)
+
+        MissionControl::Jobs.delay_between_bulk_operation_batches = 2
+        MissionControl::Jobs.logger = ActiveSupport::Logger.new(STDOUT)
+
+        puts "\n\nType 'jobs_help' to see how to connect to the available job servers to manage jobs\n\n"
+      end
+
+      initializer "mission_control-jobs.assets" do |app|
+        app.config.assets.paths << root.join("app/javascript")
+        app.config.assets.precompile += %w[ mission_control_jobs_manifest ]
+      end
+
+      initializer "mission_control-jobs.importmap", before: "importmap" do |app|
+        app.config.importmap.paths << root.join("config/importmap.rb")
+        app.config.importmap.cache_sweepers << root.join("app/javascript")
+      end
+    end
+  end
+end

data/lib/mission_control/jobs/errors/incompatible_adapter.rb

@@ -0,0 +1,2 @@
+class MissionControl::Jobs::Errors::IncompatibleAdapter < StandardError
+end

data/lib/mission_control/jobs/errors/resource_not_found.rb

@@ -0,0 +1,2 @@
+class MissionControl::Jobs::Errors::ResourceNotFound < StandardError
+end

data/lib/mission_control/jobs/identified_by_name.rb

@@ -0,0 +1,18 @@
+module MissionControl::Jobs::IdentifiedByName
+  extend ActiveSupport::Concern
+
+  included do
+    attr_reader :name
+    alias to_s name
+  end
+
+  def initialize(name:)
+    @name = name.to_s
+  end
+
+  def id
+    name.parameterize
+  end
+
+  alias to_param id
+end

data/lib/mission_control/jobs/identified_elements.rb

@@ -0,0 +1,23 @@
+# A collection of elements offering a Hash-like access based on
+# their +id+.
+class MissionControl::Jobs::IdentifiedElements
+  include Enumerable
+
+  delegate :[], :empty?, to: :elements
+  delegate :each, :last, :length, to: :to_a
+
+  def initialize
+    @elements = HashWithIndifferentAccess.new
+  end
+
+  def <<(item)
+    @elements[item.id] = item
+  end
+
+  def to_a
+    @elements.values
+  end
+
+  private
+    attr_reader :elements
+end

data/lib/mission_control/jobs/server/serializable.rb

@@ -0,0 +1,24 @@
+module MissionControl::Jobs::Server::Serializable
+  extend ActiveSupport::Concern
+
+  class_methods do
+    # Loads a server from a locator string with the format +<application>:<server>+. For example:
+    #
+    #   bc4:resque_chicago
+    #
+    # When the +<server>+ fragment is omitted it will return the first server for the application.
+    def from_global_id(global_id)
+      app_id, server_id = global_id.split(":")
+
+      application = MissionControl::Jobs.applications[app_id] or raise MissionControl::Jobs::Errors::ResourceNotFound, "No application with id #{app_id} found"
+      server = server_id ? application.servers[server_id] : application.servers.first
+
+      server or raise MissionControl::Jobs::Errors::ResourceNotFound, "No server for #{global_id} found"
+    end
+  end
+
+  def to_global_id
+    suffix = ":#{id}" if application.servers.length > 1
+    "#{application&.id}#{suffix}"
+  end
+end

data/lib/mission_control/jobs/server/workers.rb

@@ -0,0 +1,15 @@
+module MissionControl::Jobs::Server::Workers
+  def workers
+    queue_adapter.workers.collect do |worker|
+      MissionControl::Jobs::Worker.new(queue_adapter: queue_adapter, **worker)
+    end
+  end
+
+  def find_worker(worker_id)
+    if worker = queue_adapter.find_worker(worker_id)
+      MissionControl::Jobs::Worker.new(queue_adapter: queue_adapter, **worker)
+    else
+      raise MissionControl::Jobs::Errors::ResourceNotFound, "No worker found with ID #{worker_id}"
+    end
+  end
+end