solid_job_monitor 1.0.0

data/lib/mission_control/jobs/engine.rb

@@ -0,0 +1,118 @@
+require "importmap-rails"
+require "turbo-rails"
+require "stimulus-rails"
+
+module MissionControl
+  module Jobs
+    class Engine < ::Rails::Engine
+      isolate_namespace MissionControl::Jobs
+
+      rake_tasks do
+        load "mission_control/jobs/tasks.rb"
+      end
+
+      initializer "mission_control-jobs.middleware" do |app|
+        if app.config.api_only
+          config.middleware.use ActionDispatch::Flash
+          config.middleware.use ::Rack::MethodOverride
+        end
+      end
+
+      config.mission_control = ActiveSupport::OrderedOptions.new unless config.try(:mission_control)
+      config.mission_control.jobs = ActiveSupport::OrderedOptions.new
+
+      config.before_initialize do
+        config.mission_control.jobs.applications = MissionControl::Jobs::Applications.new
+        config.mission_control.jobs.backtrace_cleaner ||= Rails::BacktraceCleaner.new
+
+        config.mission_control.jobs.each do |key, value|
+          MissionControl::Jobs.public_send("#{key}=", value)
+        end
+
+        if MissionControl::Jobs.adapters.empty?
+          MissionControl::Jobs.adapters << (config.active_job.queue_adapter || :async)
+        end
+      end
+
+      initializer "mission_control-jobs.http_basic_auth" do |app|
+        MissionControl::Jobs.http_basic_auth_user ||= app.credentials.dig(:mission_control, :http_basic_auth_user)
+        MissionControl::Jobs.http_basic_auth_password ||= app.credentials.dig(:mission_control, :http_basic_auth_password)
+      end
+
+      initializer "mission_control-jobs.active_job.extensions", before: "active_job.set_configs" do
+        ActiveSupport.on_load :active_job do
+          include ActiveJob::Querying
+          include ActiveJob::Executing
+          include ActiveJob::Failed
+          include ActiveJob::ProgressReporting
+          ActiveJob.extend ActiveJob::Querying::Root
+        end
+      end
+
+      initializer "mission_control-jobs.append_migrations" do |app|
+        next if app.root.to_s.start_with?(root.to_s)
+
+        config.paths["db/migrate"].expanded.each do |expanded_path|
+          app.config.paths["db/migrate"] << expanded_path
+        end
+      end
+
+      config.before_initialize do
+        if MissionControl::Jobs.adapters.include?(:resque)
+          require "resque/thread_safe_redis"
+          ActiveJob::QueueAdapters::ResqueAdapter.prepend ActiveJob::QueueAdapters::ResqueExt
+          Resque.prepend Resque::ThreadSafeRedis
+        end
+
+        if MissionControl::Jobs.adapters.include?(:solid_queue)
+          ActiveJob::QueueAdapters::SolidQueueAdapter.prepend ActiveJob::QueueAdapters::SolidQueueExt
+        end
+
+        ActiveJob::QueueAdapters::AsyncAdapter.include ActiveJob::QueueAdapters::AsyncExt
+      end
+
+      config.after_initialize do |app|
+        if MissionControl::Jobs.applications.empty?
+          queue_adapters_by_name = MissionControl::Jobs.adapters.each_with_object({}) do |adapter, hsh|
+            hsh[adapter] = ActiveJob::QueueAdapters.lookup(adapter).new
+          end
+
+          MissionControl::Jobs.applications.add(app.class.module_parent.name, queue_adapters_by_name)
+        end
+      end
+
+      console do
+        require "irb"
+
+        IRB::Command.register :connect_to, Console::ConnectTo
+        IRB::Command.register :jobs_help, Console::JobsHelp
+
+        IRB::Context.prepend(MissionControl::Jobs::Console::Context)
+
+        MissionControl::Jobs.delay_between_bulk_operation_batches = 2
+        MissionControl::Jobs.logger = ActiveSupport::Logger.new(STDOUT)
+
+        if MissionControl::Jobs.show_console_help
+          puts "\n\nType 'jobs_help' to see how to connect to the available job servers to manage jobs\n\n"
+        end
+      end
+
+      initializer "mission_control-jobs.assets" do |app|
+        app.config.assets.paths << root.join("app/assets/stylesheets")
+        app.config.assets.paths << root.join("app/javascript")
+        app.config.assets.precompile += %w[ mission_control_jobs_manifest ]
+      end
+
+      initializer "mission_control-jobs.importmap", after: "importmap" do |app|
+        MissionControl::Jobs.importmap.draw(root.join("config/importmap.rb"))
+        if app.config.importmap.sweep_cache && app.config.reloading_enabled?
+          MissionControl::Jobs.importmap.cache_sweeper(watches: root.join("app/javascript"))
+
+          ActiveSupport.on_load(:action_controller_base) do
+            before_action { MissionControl::Jobs.importmap.cache_sweeper.execute_if_updated }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mission_control/jobs/errors/incompatible_adapter.rb

@@ -0,0 +1,2 @@
+class MissionControl::Jobs::Errors::IncompatibleAdapter < StandardError
+end

data/lib/mission_control/jobs/errors/resource_not_found.rb

@@ -0,0 +1,2 @@
+class MissionControl::Jobs::Errors::ResourceNotFound < StandardError
+end

data/lib/mission_control/jobs/i18n_config.rb

@@ -0,0 +1,17 @@
+class MissionControl::Jobs::I18nConfig < ::I18n::Config
+  AVAILABLE_LOCALES = [ :en ]
+  AVAILABLE_LOCALES_SET = [ :en, "en" ]
+  DEFAULT_LOCALE = :en
+
+  def available_locales
+    AVAILABLE_LOCALES
+  end
+
+  def available_locales_set
+    AVAILABLE_LOCALES_SET
+  end
+
+  def default_locale
+    DEFAULT_LOCALE
+  end
+end

data/lib/mission_control/jobs/identified_by_name.rb

@@ -0,0 +1,18 @@
+module MissionControl::Jobs::IdentifiedByName
+  extend ActiveSupport::Concern
+
+  included do
+    attr_reader :name
+    alias to_s name
+  end
+
+  def initialize(name:)
+    @name = name.to_s
+  end
+
+  def id
+    name.parameterize
+  end
+
+  alias to_param id
+end

data/lib/mission_control/jobs/identified_elements.rb

@@ -0,0 +1,23 @@
+# A collection of elements offering a Hash-like access based on
+# their +id+.
+class MissionControl::Jobs::IdentifiedElements
+  include Enumerable
+
+  delegate :[], :empty?, to: :elements
+  delegate :each, :last, :length, to: :to_a
+
+  def initialize
+    @elements = HashWithIndifferentAccess.new
+  end
+
+  def <<(item)
+    @elements[item.id] = item
+  end
+
+  def to_a
+    @elements.values
+  end
+
+  private
+    attr_reader :elements
+end

data/lib/mission_control/jobs/job_stopped_error.rb

@@ -0,0 +1,5 @@
+class MissionControl::Jobs::JobStoppedError < StandardError
+  def initialize(job_id = nil)
+    super(job_id ? "Job #{job_id} was stopped" : "Job was stopped")
+  end
+end

data/lib/mission_control/jobs/progress/null_store.rb

@@ -0,0 +1,25 @@
+class MissionControl::Jobs::Progress::NullStore
+  def upsert(**)
+    nil
+  end
+
+  def find(_job_id)
+    nil
+  end
+
+  def find_multi(_job_ids)
+    {}
+  end
+
+  def request_stop(_job_id)
+    nil
+  end
+
+  def stop_requested?(_job_id)
+    false
+  end
+
+  def complete(_job_id)
+    nil
+  end
+end

data/lib/mission_control/jobs/progress/reporter.rb

@@ -0,0 +1,63 @@
+class MissionControl::Jobs::Progress::Reporter
+  def initialize(job_id)
+    @job_id = job_id
+    @current = 0
+    @total = nil
+  end
+
+  def register(total:, message: nil)
+    check_stopped!
+    @total = total
+    @current = 0
+    persist(percent: computed_percent(0), message: message)
+  end
+
+  def update(current: nil, total: nil, percent: nil, message: nil)
+    check_stopped!
+    @current = current unless current.nil?
+    @total = total unless total.nil?
+    persist(percent: computed_percent(percent), message: message)
+  end
+
+  def increment(by: 1)
+    check_stopped!
+    @current = @current.to_i + by
+    persist(percent: computed_percent(nil))
+  end
+
+  def check_stopped!
+    raise MissionControl::Jobs::JobStoppedError.new(@job_id) if store.stop_requested?(@job_id)
+  end
+
+  def complete
+    store.complete(@job_id)
+  end
+
+  private
+    def persist(percent:, message: nil)
+      store.upsert(
+        job_id: @job_id,
+        current: @current.to_i,
+        total: @total,
+        percent: percent,
+        message: message
+      )
+    end
+
+    def computed_percent(explicit)
+      return clamp_percent(explicit) unless explicit.nil?
+      return nil if @total.nil? || @total.to_i <= 0
+
+      clamp_percent(((@current.to_f / @total) * 100).round)
+    end
+
+    def clamp_percent(value)
+      return nil if value.nil?
+
+      value.to_i.clamp(0, 100)
+    end
+
+    def store
+      MissionControl::Jobs::Progress::Store.current
+    end
+end

data/lib/mission_control/jobs/progress/store.rb

@@ -0,0 +1,72 @@
+class MissionControl::Jobs::Progress::Store
+  class << self
+    def current
+      @current ||= available? ? new : null_store
+    end
+
+    def available?
+      return @available unless @available.nil?
+
+      @available = MissionControl::Jobs::JobProgress.table_exists?
+    rescue ActiveRecord::ActiveRecordError
+      @available = false
+    end
+
+    def reset!
+      @current = nil
+      @available = nil
+    end
+
+    private
+      def null_store
+        warn_if_missing
+        MissionControl::Jobs::Progress::NullStore.new
+      end
+
+      def warn_if_missing
+        return if @warned
+
+        @warned = true
+        logger = (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) || MissionControl::Jobs.logger
+        logger&.warn(
+          "[solid_job_monitor] mission_control_job_progresses table is missing. " \
+            "Run rails db:migrate to enable job progress and cooperative stop."
+        )
+      end
+  end
+
+  def upsert(job_id:, current: 0, total: nil, percent: nil, message: nil)
+    record = MissionControl::Jobs::JobProgress.find_or_initialize_by(job_id: job_id)
+    record.current = current
+    record.total = total unless total.nil?
+    record.percent = percent unless percent.nil?
+    record.message = message unless message.nil?
+    record.save!
+    record
+  end
+
+  def find(job_id)
+    MissionControl::Jobs::JobProgress.find_by(job_id: job_id)
+  end
+
+  def find_multi(job_ids)
+    return {} if job_ids.blank?
+
+    MissionControl::Jobs::JobProgress.where(job_id: job_ids).index_by(&:job_id)
+  end
+
+  def request_stop(job_id)
+    record = MissionControl::Jobs::JobProgress.find_or_initialize_by(job_id: job_id)
+    record.stop_requested = true
+    record.save!
+    record
+  end
+
+  def stop_requested?(job_id)
+    MissionControl::Jobs::JobProgress.where(job_id: job_id, stop_requested: true).exists?
+  end
+
+  def complete(job_id)
+    MissionControl::Jobs::JobProgress.where(job_id: job_id).delete_all
+  end
+end

data/lib/mission_control/jobs/progress.rb

@@ -0,0 +1,17 @@
+module MissionControl::Jobs::Progress
+  def self.available?
+    Store.available?
+  end
+
+  def self.find(job_id)
+    Store.current.find(job_id)
+  end
+
+  def self.find_multi(job_ids)
+    Store.current.find_multi(job_ids)
+  end
+
+  def self.request_stop(job_id)
+    Store.current.request_stop(job_id)
+  end
+end

data/lib/mission_control/jobs/server/recurring_tasks.rb

@@ -0,0 +1,15 @@
+module MissionControl::Jobs::Server::RecurringTasks
+  def recurring_tasks
+    queue_adapter.recurring_tasks.collect do |task|
+      MissionControl::Jobs::RecurringTask.new(queue_adapter: queue_adapter, **task)
+    end.sort_by(&:id)
+  end
+
+  def find_recurring_task(task_id)
+    if task = queue_adapter.find_recurring_task(task_id)
+      MissionControl::Jobs::RecurringTask.new(queue_adapter: queue_adapter, **task)
+    else
+      raise MissionControl::Jobs::Errors::ResourceNotFound, "Recurring task with id '#{task_id}' not found"
+    end
+  end
+end

data/lib/mission_control/jobs/server/serializable.rb

@@ -0,0 +1,24 @@
+module MissionControl::Jobs::Server::Serializable
+  extend ActiveSupport::Concern
+
+  class_methods do
+    # Loads a server from a locator string with the format +<application>:<server>+. For example:
+    #
+    #   bc4:resque_chicago
+    #
+    # When the +<server>+ fragment is omitted it will return the first server for the application.
+    def from_global_id(global_id)
+      app_id, server_id = global_id.split(":")
+
+      application = MissionControl::Jobs.applications[app_id] or raise MissionControl::Jobs::Errors::ResourceNotFound, "No application with id #{app_id} found"
+      server = server_id ? application.servers[server_id] : application.servers.first
+
+      server or raise MissionControl::Jobs::Errors::ResourceNotFound, "No server for #{global_id} found"
+    end
+  end
+
+  def to_global_id
+    suffix = ":#{id}" if application.servers.many?
+    "#{application&.id}#{suffix}"
+  end
+end

data/lib/mission_control/jobs/server/workers.rb

@@ -0,0 +1,13 @@
+module MissionControl::Jobs::Server::Workers
+  def workers_relation
+    MissionControl::Jobs::WorkersRelation.new(queue_adapter: queue_adapter)
+  end
+
+  def find_worker(worker_id)
+    if worker = queue_adapter.find_worker(worker_id)
+      MissionControl::Jobs::Worker.new(queue_adapter: queue_adapter, **worker)
+    else
+      raise MissionControl::Jobs::Errors::ResourceNotFound, "Worker with id '#{worker_id}' not found"
+    end
+  end
+end

data/lib/mission_control/jobs/server.rb

@@ -0,0 +1,27 @@
+require "active_job/queue_adapter"
+
+class MissionControl::Jobs::Server
+  include MissionControl::Jobs::IdentifiedByName
+  include Serializable, RecurringTasks, Workers
+
+  attr_reader :name, :queue_adapter, :application, :backtrace_cleaner
+
+  def initialize(name:, queue_adapter:, application:, backtrace_cleaner: nil)
+    super(name: name)
+    @queue_adapter = queue_adapter
+    @application = application
+    @backtrace_cleaner = backtrace_cleaner || MissionControl::Jobs.backtrace_cleaner
+  end
+
+  def activating(&block)
+    previous_adapter = ActiveJob::Base.current_queue_adapter
+    ActiveJob::Base.current_queue_adapter = queue_adapter
+    queue_adapter.activating(&block)
+  ensure
+    ActiveJob::Base.current_queue_adapter = previous_adapter
+  end
+
+  def queue_adapter_name
+    ActiveJob.adapter_name(queue_adapter).underscore.to_sym
+  end
+end

data/lib/mission_control/jobs/tasks.rb

@@ -0,0 +1,8 @@
+namespace :mission_control do
+  namespace :jobs do
+    desc "Configure HTTP Basic Authentication"
+    task "authentication:configure" => :environment do
+      MissionControl::Jobs::Authentication.configure
+    end
+  end
+end

data/lib/mission_control/jobs/version.rb

@@ -0,0 +1,5 @@
+module MissionControl
+  module Jobs
+    VERSION = "1.0.0"
+  end
+end

data/lib/mission_control/jobs/workers_relation.rb

@@ -0,0 +1,78 @@
+# A relation of workers.
+#
+# Relations are enumerable, so you can use +Enumerable+ methods on them.
+# Notice however that using these methods will imply loading all the relation
+# in memory, which could introduce performance concerns.
+class MissionControl::Jobs::WorkersRelation
+  include Enumerable
+
+  attr_accessor :offset_value, :limit_value
+
+  delegate :last, :[], :to_s, :reverse, to: :to_a
+
+  ALL_WORKERS_LIMIT = 100_000_000 # When no limit value it defaults to "all workers"
+
+  def initialize(queue_adapter:)
+    @queue_adapter = queue_adapter
+
+    set_defaults
+  end
+
+  def offset(offset)
+    clone_with offset_value: offset
+  end
+
+  def limit(limit)
+    clone_with limit_value: limit
+  end
+
+  def each(&block)
+    workers.each(&block)
+  end
+
+  def reload
+    @count = @workers = nil
+    self
+  end
+
+  def count
+    if loaded?
+      to_a.length
+    else
+      query_count
+    end
+  end
+
+  def empty?
+    count == 0
+  end
+
+  alias length count
+  alias size count
+
+  private
+    def set_defaults
+      self.offset_value = 0
+      self.limit_value = ALL_WORKERS_LIMIT
+    end
+
+    def workers
+      @workers ||= @queue_adapter.fetch_workers(self)
+    end
+
+    def query_count
+      @count ||= @queue_adapter.count_workers(self)
+    end
+
+    def loaded?
+      !@workers.nil?
+    end
+
+    def clone_with(**properties)
+      dup.reload.tap do |relation|
+        properties.each do |key, value|
+          relation.send("#{key}=", value)
+        end
+      end
+    end
+end

data/lib/mission_control/jobs.rb

@@ -0,0 +1,43 @@
+require "mission_control/jobs/version"
+require "mission_control/jobs/engine"
+
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.new
+loader.inflector = Zeitwerk::GemInflector.new(__FILE__)
+loader.push_dir(File.expand_path("..", __dir__))
+loader.ignore("#{File.expand_path("..", __dir__)}/resque")
+loader.ignore("#{File.expand_path("..", __dir__)}/solid_job_monitor.rb")
+loader.ignore("#{File.expand_path("..", __dir__)}/mission_control/jobs/tasks.rb")
+loader.ignore("#{File.expand_path("..", __dir__)}/generators")
+loader.setup
+
+module MissionControl
+  module Jobs
+    mattr_accessor :adapters, default: Set.new
+    mattr_accessor :applications, default: MissionControl::Jobs::Applications.new
+    mattr_accessor :base_controller_class, default: "::ApplicationController"
+
+    mattr_accessor :internal_query_count_limit, default: 500_000 # Hard limit to keep unlimited count queries fast enough
+    mattr_accessor :delay_between_bulk_operation_batches, default: 0
+    mattr_accessor :scheduled_job_delay_threshold, default: 1.minute
+
+    mattr_accessor :logger, default: ActiveSupport::Logger.new(nil)
+
+    mattr_accessor :show_console_help, default: true
+    mattr_accessor :backtrace_cleaner
+    mattr_accessor :back_to_main_app_path
+
+    mattr_accessor :importmap, default: Importmap::Map.new
+
+    mattr_accessor :http_basic_auth_user
+    mattr_accessor :http_basic_auth_password
+    mattr_accessor :http_basic_auth_enabled, default: true
+
+    mattr_accessor :filter_arguments, default: []
+
+    def self.job_arguments_filter
+      MissionControl::Jobs::ArgumentsFilter.new(filter_arguments)
+    end
+  end
+end

data/lib/resque/thread_safe_redis.rb

@@ -0,0 +1,34 @@
+# Set and access +Resque.redis+ in a thread-safe way.
+module Resque::ThreadSafeRedis
+  thread_mattr_accessor :thread_resque_override
+
+  def self.resque_override
+    self.thread_resque_override ||= ResqueOverride.new
+  end
+
+  def redis
+    Resque::ThreadSafeRedis.resque_override.data_store_override || super
+  end
+  alias :data_store :redis
+
+  def with_per_thread_redis_override(redis_instance, &block)
+    Resque::ThreadSafeRedis.resque_override.enable_with(redis_instance, &block)
+  end
+
+  class ResqueOverride
+    include Resque
+
+    attr_accessor :data_store_override
+
+    def enable_with(server, &block)
+      previous_redis, previous_data_store_override = redis, data_store_override
+      self.redis = server
+      self.data_store_override = @data_store
+
+      block.call
+    ensure
+      self.redis = previous_redis
+      self.data_store_override = previous_data_store_override
+    end
+  end
+end

data/lib/solid_job_monitor.rb

@@ -0,0 +1,6 @@
+require "mission_control/jobs"
+
+# The gem is named solid_job_monitor; the engine remains MissionControl::Jobs.
+module SolidJobMonitor
+  VERSION = MissionControl::Jobs::VERSION
+end