mission_control-jobs 0.1.0

data/lib/active_job/jobs_relation.rb

@@ -0,0 +1,300 @@
+# A relation of jobs that can be filtered and acted on.
+#
+# Relations of jobs are normally fetched via +ActiveJob::Base.jobs+
+# or through a given queue (+ActiveJob::Queue#jobs+).
+#
+# This class offers a fluid interface to query a subset of jobs. For
+# example:
+#
+#   queue = ActiveJob::Base.queues[:default]
+#   queue.jobs.limit(10).where(job_class_name: "DummyJob").last
+#
+# Relations are enumerable, so you can use +Enumerable+ methods on them.
+# Notice however that using these methods will imply loading all the relation
+# in memory, which could introduce performance concerns.
+#
+# Internally, +ActiveJob+ will always use paginated queries to the underlying
+# queue adapter. The page size can be controlled via +config.active_job.default_page_size+
+# (1000 by default).
+#
+# There are additional performance concerns depending on the configured
+# adapter. Please check +ActiveJob::Relation#where+, +ActiveJob::Relation#count+.
+class ActiveJob::JobsRelation
+  include Enumerable
+
+  STATUSES = %i[ pending failed in_progress blocked scheduled finished ]
+  FILTERS = %i[ queue_name job_class_name ]
+
+  PROPERTIES = %i[ queue_name status offset_value limit_value job_class_name worker_id ]
+  attr_reader *PROPERTIES, :default_page_size
+
+  delegate :last, :[], :reverse, to: :to_a
+  delegate :logger, to: MissionControl::Jobs
+
+  ALL_JOBS_LIMIT = 100_000_000 # When no limit value it defaults to "all jobs"
+
+  def initialize(queue_adapter: ActiveJob::Base.queue_adapter, default_page_size: ActiveJob::Base.default_page_size)
+    @queue_adapter = queue_adapter
+    @default_page_size = default_page_size
+
+    set_defaults
+  end
+
+  # Returns a +ActiveJob::JobsRelation+ with the configured filtering options.
+  #
+  # === Options
+  #
+  # * <tt>:job_class_name</tt> - To only include the jobs of a given class.
+  #   Depending on the configured queue adapter, this will perform the
+  #   filtering in memory, which could introduce performance concerns
+  #   for large sets of jobs.
+  # * <tt>:queue_name</tt> - To only include the jobs in the provided queue.
+  # * <tt>:worker_id</tt> - To only include the jobs processed by the provided worker.
+  def where(job_class_name: nil, queue_name: nil, worker_id: nil)
+    # Remove nil arguments to avoid overriding parameters when concatenating +where+ clauses
+    arguments = { job_class_name: job_class_name, queue_name: queue_name, worker_id: worker_id }.compact.collect { |key, value| [ key, value.to_s ] }.to_h
+    clone_with **arguments
+  end
+
+  def with_status(status)
+    if status.to_sym.in? STATUSES
+      clone_with status: status.to_sym
+    else
+      self
+    end
+  end
+
+  STATUSES.each do |status|
+    define_method status do
+      with_status(status)
+    end
+
+    define_method "#{status}?" do
+      self.status == status
+    end
+  end
+
+  # Sets an offset for the jobs-fetching query. The first position is 0.
+  def offset(offset)
+    clone_with offset_value: offset
+  end
+
+  # Sets the max number of jobs to fetch in the query.
+  def limit(limit)
+    clone_with limit_value: limit
+  end
+
+  # Returns the number of jobs in the relation.
+  #
+  # When filtering jobs, if the adapter doesn't support the filter(s)
+  # directly, this will load all the jobs in memory to filter them.
+  def count
+    if loaded? || filtering_needed?
+      to_a.length
+    else
+      query_count
+    end
+  end
+
+  alias length count
+  alias size count
+
+  def empty?
+    count == 0
+  end
+
+  def to_s
+    properties_with_values = PROPERTIES.collect do |name|
+      value = public_send(name)
+      "#{name}: #{value}" unless value.nil?
+    end.compact.join(", ")
+    "<Jobs with [#{properties_with_values}]> (loaded: #{loaded?})"
+  end
+
+  alias inspect to_s
+
+  def each(&block)
+    loaded_jobs&.each(&block) || load_jobs(&block)
+  end
+
+  # Retry all the jobs in the queue.
+  #
+  # This operation is only valid for sets of failed jobs. It will
+  # raise an error +ActiveJob::Errors::InvalidOperation+ otherwise.
+  def retry_all
+    ensure_failed_status
+    queue_adapter.retry_all_jobs(self)
+    nil
+  end
+
+  # Retry the provided job.
+  #
+  # This operation is only valid for sets of failed jobs. It will
+  # raise an error +ActiveJob::Errors::InvalidOperation+ otherwise.
+  def retry_job(job)
+    ensure_failed_status
+    queue_adapter.retry_job(job, self)
+  end
+
+  # Discard all the jobs in the relation.
+  def discard_all
+    queue_adapter.discard_all_jobs(self)
+    nil
+  end
+
+  # Discard the provided job.
+  def discard_job(job)
+    queue_adapter.discard_job(job, self)
+  end
+
+  # Find a job by id.
+  #
+  # Returns nil when not found.
+  def find_by_id(job_id)
+    queue_adapter.find_job(job_id, self)
+  end
+
+  # Find a job by id.
+  #
+  # Raises +ActiveJob::Errors::JobNotFoundError+ when not found.
+  def find_by_id!(job_id)
+    queue_adapter.find_job(job_id, self) or raise ActiveJob::Errors::JobNotFoundError.new(job_id, self)
+  end
+
+  # Returns an array of jobs class names in the first +from_first+ jobs.
+  def job_class_names(from_first: 500)
+    first(from_first).collect(&:job_class_name).uniq
+  end
+
+  def reload
+    @count = nil
+    @loaded_jobs = nil
+    @filters = nil
+
+    self
+  end
+
+  def in_batches(of: default_page_size, order: :asc, &block)
+    validate_looping_in_batches_is_possible
+
+    case order
+    when :asc
+      in_ascending_batches(of: of, &block)
+    when :desc
+      in_descending_batches(of: of, &block)
+    else
+      raise "Unsupported order: #{order}. Valid values: :asc, :desc."
+    end
+  end
+
+  def paginated?
+    offset_value > 0 || limit_value_provided?
+  end
+
+  def limit_value_provided?
+    limit_value.present? && limit_value != ActiveJob::JobsRelation::ALL_JOBS_LIMIT
+  end
+
+  def filtering_needed?
+    filters.any?
+  end
+
+  private
+    attr_reader :queue_adapter, :loaded_jobs
+    attr_writer *PROPERTIES
+
+    def set_defaults
+      self.offset_value = 0
+      self.limit_value = ALL_JOBS_LIMIT
+    end
+
+    def clone_with(**properties)
+      dup.reload.tap do |relation|
+        properties.each do |key, value|
+          relation.send("#{key}=", value)
+        end
+      end
+    end
+
+    def query_count
+      @count ||= queue_adapter.jobs_count(self)
+    end
+
+    def load_jobs
+      @loaded_jobs = []
+      perform_each do |job|
+        @loaded_jobs << job
+        yield job
+      end
+    end
+
+    def perform_each
+      current_offset = offset_value
+      pending_count = limit_value || Float::INFINITY
+
+      begin
+        limit = [ pending_count, default_page_size ].min
+        page = offset(current_offset).limit(limit)
+        jobs = queue_adapter.fetch_jobs(page)
+        finished = jobs.empty?
+        jobs = filter(jobs) if filtering_needed?
+        Array(jobs).each { |job| yield job }
+        current_offset += limit
+        pending_count -= jobs.length
+      end until finished || pending_count.zero?
+    end
+
+    def loaded?
+      !@loaded_jobs.nil?
+    end
+
+    # Filtering for not natively supported filters is performed in memory
+    def filter(jobs)
+      jobs.filter { |job| satisfy_filter?(job) }
+    end
+
+    def satisfy_filter?(job)
+      filters.all? { |property| public_send(property) == job.public_send(property) }
+    end
+
+    def filters
+      @filters ||= FILTERS.select { |property| public_send(property).present? && !queue_adapter.supports_filter?(self, property) }
+    end
+
+    def ensure_failed_status
+      raise ActiveJob::Errors::InvalidOperation, "This operation can only be performed on failed jobs, but these jobs are #{status}" unless failed?
+    end
+
+    def validate_looping_in_batches_is_possible
+      raise ActiveJob::Errors::InvalidOperation, "Looping in batches is not compatible with providing offset or limit" if paginated?
+    end
+
+    def in_ascending_batches(of:)
+      current_offset = 0
+      max = count
+      begin
+        page = offset(current_offset).limit(of)
+        current_offset += of
+        logger.info page
+        yield page
+        wait_batch_delay
+      end until current_offset >= max
+    end
+
+    def in_descending_batches(of:)
+      current_offset = count - of
+
+      begin
+        limit = current_offset < 0 ? of + current_offset : of
+        page = offset([ current_offset, 0 ].max).limit(limit)
+        current_offset -= of
+        logger.info page
+        yield page
+        wait_batch_delay
+      end until current_offset + of <= 0
+    end
+
+    def wait_batch_delay
+      sleep MissionControl::Jobs.delay_between_bulk_operation_batches if MissionControl::Jobs.delay_between_bulk_operation_batches.to_i > 0
+    end
+end

data/lib/active_job/querying.rb

@@ -0,0 +1,44 @@
+module ActiveJob::Querying
+  extend ActiveSupport::Concern
+
+  included do
+    # ActiveJob will use pagination internally when fetching relations of jobs. This
+    # parameter sets the max amount of jobs to fetch in each data store query.
+    class_attribute :default_page_size, default: 1000
+  end
+
+  class_methods do
+    # Returns the list of queues.
+    #
+    # See +ActiveJob::Queues+
+    def queues
+      ActiveJob::Queues.new(fetch_queues)
+    end
+
+    def jobs
+      ActiveJob::JobsRelation.new(queue_adapter: queue_adapter, default_page_size: default_page_size)
+    end
+
+    private
+      def fetch_queues
+        queue_adapter.queues.collect do |queue|
+          ActiveJob::Queue.new(queue[:name], size: queue[:size], active: queue[:active], queue_adapter: queue_adapter)
+        end
+      end
+  end
+
+  def queue
+    self.class.queues[queue_name]
+  end
+
+  # Top-level query methods added to `ActiveJob`
+  module Root
+    def queues
+      ActiveJob::Base.queues
+    end
+
+    def jobs
+      ActiveJob::Base.jobs
+    end
+  end
+end

data/lib/active_job/queue.rb

@@ -0,0 +1,62 @@
+# A queue of jobs
+class ActiveJob::Queue
+  attr_reader :name
+
+  def initialize(name, size: nil, active: nil, queue_adapter: ActiveJob::Base.queue_adapter)
+    @name = name
+    @queue_adapter = queue_adapter
+
+    @size = size
+    @active = active
+  end
+
+  def size
+    @size ||= queue_adapter.queue_size(name)
+  end
+
+  alias length size
+
+  def clear
+    queue_adapter.clear_queue(name)
+  end
+
+  def empty?
+    size == 0
+  end
+
+  def pause
+    queue_adapter.pause_queue(name)
+  end
+
+  def resume
+    queue_adapter.resume_queue(name)
+  end
+
+  def paused?
+    !active?
+  end
+
+  def active?
+    return @active unless @active.nil?
+    @active = !queue_adapter.queue_paused?(name)
+  end
+
+  # Return an +ActiveJob::JobsRelation+ with the pending jobs in the queue.
+  def jobs
+    ActiveJob::JobsRelation.new(queue_adapter: queue_adapter).pending.where(queue_name: name)
+  end
+
+  def reload
+    @active = @size = nil
+    self
+  end
+
+  def id
+    name.parameterize
+  end
+
+  alias to_param id
+
+  private
+    attr_reader :queue_adapter
+end

data/lib/active_job/queue_adapters/resque_ext.rb

@@ -0,0 +1,300 @@
+module ActiveJob::QueueAdapters::ResqueExt
+  include MissionControl::Jobs::Adapter
+
+  def initialize(redis = Resque.redis)
+    super()
+    @redis = redis
+  end
+
+  def activating(&block)
+    Resque.with_per_thread_redis_override(redis, &block)
+  end
+
+  def queues
+    queues = queue_names
+    active_statuses = []
+    counts = []
+
+    redis.multi do |multi|
+      queues.each do |queue_name|
+        active_statuses << multi.mget("pause:queue:#{queue_name}", "pause:all")
+        counts << multi.llen("queue:#{queue_name}")
+      end
+    end
+
+    queues.collect.with_index do |queue_name, index|
+      { name: queue_name, active: active_statuses[index].value.compact.empty?, size: counts[index].value }
+    end
+  end
+
+  def queue_size(queue_name)
+    Resque.size queue_name
+  end
+
+  def clear_queue(queue_name)
+    Resque.remove_queue(queue_name)
+  end
+
+  def pause_queue(queue_name)
+    ResquePauseHelper.pause(queue_name)
+  end
+
+  def resume_queue(queue_name)
+    ResquePauseHelper.unpause(queue_name)
+  end
+
+  def queue_paused?(queue_name)
+    ResquePauseHelper.paused?(queue_name)
+  end
+
+  def supported_filters(jobs_relation)
+    if jobs_relation.pending? then [ :queue_name ]
+    else []
+    end
+  end
+
+  def jobs_count(jobs_relation)
+    resque_jobs_for(jobs_relation).count
+  end
+
+  def fetch_jobs(jobs_relation)
+    resque_jobs_for(jobs_relation).all
+  end
+
+  def retry_all_jobs(jobs_relation)
+    resque_jobs_for(jobs_relation).retry_all
+  end
+
+  def retry_job(job, jobs_relation)
+    resque_jobs_for(jobs_relation).retry_job(job)
+  end
+
+  def discard_all_jobs(jobs_relation)
+    resque_jobs_for(jobs_relation).discard_all
+  end
+
+  def discard_job(job, jobs_relation)
+    resque_jobs_for(jobs_relation).discard(job)
+  end
+
+  def find_job(job_id, jobs_relation)
+    resque_jobs_for(jobs_relation).find_job(job_id)
+  end
+
+  private
+    attr_reader :redis
+
+    def queue_names
+      Resque.queues
+    end
+
+    def resque_jobs_for(jobs_relation)
+      ResqueJobs.new(jobs_relation, redis: redis)
+    end
+
+    class ResqueJobs
+      attr_reader :jobs_relation
+
+      delegate :default_page_size, :paginated?, :limit_value_provided?, to: :jobs_relation
+
+      def initialize(jobs_relation, redis:)
+        @jobs_relation = jobs_relation
+        @redis = redis
+      end
+
+      def count
+        if paginated?
+          count_fetched_jobs # no direct way of counting jobs
+        else
+          direct_jobs_count
+        end
+      end
+
+      def all
+        @all ||= fetch_resque_jobs.collect.with_index { |resque_job, index| deserialize_resque_job(resque_job, index) if resque_job.is_a?(Hash) }.compact
+      end
+
+      def retry_all
+        if use_batches?
+          retry_all_in_batches
+        else
+          retry_jobs(jobs_relation.to_a.reverse)
+        end
+      end
+
+      def retry_job(job)
+        # Not named just +retry+ because it collides with reserved Ruby keyword.
+        resque_requeue_and_discard(job)
+      end
+
+      def discard_all
+        if jobs_relation.failed? && targeting_all_jobs?
+          clear_failed_queue
+        else
+          discard_all_one_by_one
+        end
+      end
+
+      def discard(job)
+        redis.multi do |multi|
+          multi.lset(queue_redis_key, job.position, SENTINEL)
+          multi.lrem(queue_redis_key, 1, SENTINEL)
+        end
+      rescue Redis::CommandError => error
+        handle_resque_job_error(job, error)
+      end
+
+      def find_job(job_id)
+        jobs_by_id[job_id]
+      end
+
+      private
+        attr_reader :redis
+
+        SENTINEL = "" # See +Resque::Datastore#remove_from_failed_queue+
+
+        # Redis transactions severely speed up operations, specially when the network latency is high.
+        # We limit the transaction size because large batches can result in redis timeout errors.
+        MAX_REDIS_TRANSACTION_SIZE = 100
+
+        def targeting_all_jobs?
+          !paginated? && !jobs_relation.filtering_needed?
+        end
+
+        def fetch_resque_jobs
+          if jobs_relation.failed? || jobs_relation.queue_name.blank?
+            fetch_failed_resque_jobs
+          else
+            fetch_queue_resque_jobs
+          end
+        end
+
+        def fetch_failed_resque_jobs
+          Array.wrap(Resque::Failure.all(jobs_relation.offset_value, jobs_relation.limit_value))
+        end
+
+        def fetch_queue_resque_jobs
+          unless jobs_relation.queue_name.present?
+            raise ActiveJob::Errors::QueryError, "This adapter requires a queue name unless fetching failed jobs"
+          end
+          Array.wrap(Resque.peek(jobs_relation.queue_name, jobs_relation.offset_value, jobs_relation.limit_value))
+        end
+
+        def deserialize_resque_job(resque_job_hash, index)
+          args_hash = resque_job_hash.dig("payload", "args") || resque_job_hash.dig("args")
+          ActiveJob::JobProxy.new(args_hash&.first).tap do |job|
+            job.last_execution_error = execution_error_from_resque_job(resque_job_hash)
+            job.raw_data = resque_job_hash
+            job.position = jobs_relation.offset_value + index
+            job.failed_at = resque_job_hash["failed_at"]&.to_datetime
+            job.status = job.failed_at.present? ? :failed : :pending
+          end
+        end
+
+        def execution_error_from_resque_job(resque_job_hash)
+          if resque_job_hash["exception"].present?
+            ActiveJob::ExecutionError.new \
+              error_class: resque_job_hash["exception"],
+              message: resque_job_hash["error"],
+              backtrace: resque_job_hash["backtrace"]
+          end
+        end
+
+        def direct_jobs_count
+          jobs_relation.failed? ? failed_jobs_count : pending_jobs_count
+        end
+
+        def pending_jobs_count
+          Resque.queue_sizes.inject(0) do |sum, (queue_name, queue_size)|
+            if jobs_relation.queue_name.blank? || jobs_relation.queue_name == queue_name
+              sum + queue_size
+            else
+              sum
+            end
+          end
+        end
+
+        def failed_jobs_count
+          Resque.data_store.num_failed
+        end
+
+        def count_fetched_jobs
+          all.size
+        end
+
+        def queue_redis_key
+          jobs_relation.failed? ? "failed" : "queue:#{jobs_relation.queue_name}"
+        end
+
+        def clear_failed_queue
+          Resque::Failure.clear("failed")
+        end
+
+        def retry_jobs(jobs)
+          in_transactional_jobs_batches(jobs) do |jobs_batch|
+            jobs_batch.each { |job| retry_job(job) }
+          end
+        end
+
+        def in_transactional_jobs_batches(jobs)
+          jobs.each_slice(MAX_REDIS_TRANSACTION_SIZE) do |jobs_batch|
+            redis.multi do |multi|
+              yield jobs_batch
+            end
+          end
+        end
+
+        def use_batches?
+          !jobs_relation.limit_value_provided? && jobs_relation.count > default_page_size
+        end
+
+        def retry_all_in_batches
+          jobs_relation.in_batches(order: :desc, &:retry_all)
+        end
+
+        def resque_requeue_and_discard(job)
+          requeue(job)
+          discard(job)
+        end
+
+        def requeue(job)
+          resque_job = job.raw_data
+          resque_job["retried_at"] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+
+          redis.lset(queue_redis_key, job.position, resque_job)
+          Resque::Job.create(resque_job["queue"], resque_job["payload"]["class"], *resque_job["payload"]["args"])
+        rescue Redis::CommandError => error
+          handle_resque_job_error(job, error)
+        end
+
+        def discard_all_one_by_one
+          if use_batches?
+            discard_all_in_batches
+          else
+            discard_jobs(jobs_relation.to_a.reverse)
+          end
+        end
+
+        def discard_jobs(jobs)
+          in_transactional_jobs_batches(jobs) do |jobs_batch|
+            jobs_batch.each { |job| discard(job) }
+          end
+        end
+
+        def discard_all_in_batches
+          jobs_relation.in_batches(order: :desc, &:discard_all)
+        end
+
+        def jobs_by_id
+          @jobs_by_id ||= all.index_by(&:job_id)
+        end
+
+        def handle_resque_job_error(job, error)
+          if error.message =~/no such key/i
+            raise ActiveJob::Errors::JobNotFoundError.new(job, jobs_relation)
+          else
+            raise error
+          end
+        end
+    end
+end