mission_control-jobs 0.1.1 → 0.2.1

data/app/views/mission_control/jobs/shared/_pagination_toolbar.html.erb

@@ -1,5 +1,5 @@
 <nav class="buttons is-right" role="navigation" aria-label="pagination">
-  <span class="mr-3"><%= jobs_page.index %> / <%= jobs_page.pages_count || "..." %></span>
-  <%= link_to "Previous page", url_for(page: jobs_page.previous_index, **jobs_filter_param), class: "pagination-previous", disabled: jobs_page.first? %>
-  <%= link_to "Next page", url_for(page: jobs_page.next_index, **jobs_filter_param), class: "pagination-next", disabled: jobs_page.last? %>
+  <span class="mr-3"><%= page.index %> / <%= page.pages_count || "..." %></span>
+  <%= link_to "Previous page", url_for(page: page.previous_index, **filter_param), class: "pagination-previous", disabled: page.first? %>
+  <%= link_to "Next page", url_for(page: page.next_index, **filter_param), class: "pagination-next", disabled: page.last? %>
 </nav>

data/app/views/mission_control/jobs/workers/_workers_page.html.erb

@@ -0,0 +1,15 @@
+<table class="workers table jobs is-hoverable is-fullwidth">
+  <tbody>
+  <thead>
+  <tr>
+    <th>Worker</th>
+    <th>Hostname</th>
+    <th style="width: 35%;">Jobs</th>
+    <th>Last heartbeat</th>
+  </tr>
+  </thead>
+
+  <%= render partial: "mission_control/jobs/workers/worker", collection: workers_page.records %>
+
+  </tbody>
+</table>

data/app/views/mission_control/jobs/workers/index.html.erb

@@ -1,17 +1,6 @@
 <% navigation(title: "Workers", section: :workers)  %>
 
-<table class="workers table jobs is-hoverable is-fullwidth">
-  <tbody>
-  <thead>
-  <tr>
-    <th>Worker</th>
-    <th>Hostname</th>
-    <th style="width: 35%;">Jobs</th>
-    <th>Last heartbeat</th>
-  </tr>
-  </thead>
+<%= render "mission_control/jobs/workers/workers_page", workers_page: @workers_page %>
 
-  <%= render partial: "mission_control/jobs/workers/worker", collection: @workers %>
+<%= render "mission_control/jobs/shared/pagination_toolbar", page: @workers_page, filter_param: {} %>
 
-  </tbody>
-</table>

data/app/views/mission_control/jobs/workers/show.html.erb

@@ -2,6 +2,13 @@
 
 <%= render "mission_control/jobs/workers/title", worker: @worker %>
 <%= render "mission_control/jobs/workers/configuration", worker: @worker %>
-<%= render "mission_control/jobs/workers/jobs", worker: @worker %>
-<%= render "mission_control/jobs/workers/raw_data", worker: @worker %>
 
+<% if @worker.jobs.empty? %>
+  <%= blank_status_notice "This worker is idle" %>
+<% else %>
+  <h2 class="subtitle">Running <%= pluralize @worker.jobs.size, "job" %></h2>
+
+  <%= render "mission_control/jobs/shared/jobs", jobs: @worker.jobs %>
+<% end %>
+
+<%= render "mission_control/jobs/workers/raw_data", worker: @worker %>

data/config/routes.rb

@@ -9,6 +9,7 @@ MissionControl::Jobs::Engine.routes.draw do
     resources :jobs, only: :show do
       resource :retry, only: :create
       resource :discard, only: :create
+      resource :dispatch, only: :create
 
       collection do
         resource :bulk_retries, only: :create
@@ -19,6 +20,7 @@ MissionControl::Jobs::Engine.routes.draw do
     resources :jobs, only: :index, path: ":status/jobs"
 
     resources :workers, only: [ :index, :show ]
+    resources :recurring_tasks, only: [ :index, :show ]
   end
 
   # Allow referencing urls without providing an application_id. It will default to the first one.
@@ -27,7 +29,5 @@ MissionControl::Jobs::Engine.routes.draw do
   resources :jobs, only: :show
   resources :jobs, only: :index, path: ":status/jobs"
 
-  resources :workers, only: [ :index, :show ]
-
   root to: "queues#index"
 end

data/lib/active_job/executing.rb

@@ -4,9 +4,8 @@ module ActiveJob::Executing
   extend ActiveSupport::Concern
 
   included do
-    attr_accessor :raw_data, :position, :finished_at, :blocked_by, :blocked_until, :worker_id, :started_at
+    attr_accessor :raw_data, :position, :finished_at, :blocked_by, :blocked_until, :worker_id, :started_at, :status
     attr_reader :serialized_arguments
-    attr_writer :status
 
     thread_cattr_accessor :current_queue_adapter
   end
@@ -25,10 +24,8 @@ module ActiveJob::Executing
     jobs_relation_for_discarding.discard_job(self)
   end
 
-  def status
-    return @status if @status.present?
-
-    failed? ? :failed : :pending
+  def dispatch
+    ActiveJob.jobs.blocked.dispatch_job(self)
   end
 
   private

data/lib/active_job/failed.rb

@@ -4,8 +4,4 @@ module ActiveJob::Failed
   included do
     attr_accessor :last_execution_error, :failed_at
   end
-
-  def failed?
-    last_execution_error.present?
-  end
 end

data/lib/active_job/job_proxy.rb

@@ -23,4 +23,10 @@ class ActiveJob::JobProxy < ActiveJob::Base
   def perform_now
     raise UnsupportedError, "A JobProxy doesn't support immediate execution, only enqueuing."
   end
+
+  ActiveJob::JobsRelation::STATUSES.each do |status|
+    define_method "#{status}?" do
+      self.status == status
+    end
+  end
 end

data/lib/active_job/jobs_relation.rb

@@ -1,12 +1,12 @@
 # A relation of jobs that can be filtered and acted on.
 #
-# Relations of jobs are normally fetched via +ActiveJob::Base.jobs+
+# Relations of jobs are normally fetched via +ActiveJob.jobs+
 # or through a given queue (+ActiveJob::Queue#jobs+).
 #
 # This class offers a fluid interface to query a subset of jobs. For
 # example:
 #
-#   queue = ActiveJob::Base.queues[:default]
+#   queue = ActiveJob.queues[:default]
 #   queue.jobs.limit(10).where(job_class_name: "DummyJob").last
 #
 # Relations are enumerable, so you can use +Enumerable+ methods on them.
@@ -25,7 +25,7 @@ class ActiveJob::JobsRelation
   STATUSES = %i[ pending failed in_progress blocked scheduled finished ]
   FILTERS = %i[ queue_name job_class_name ]
 
-  PROPERTIES = %i[ queue_name status offset_value limit_value job_class_name worker_id ]
+  PROPERTIES = %i[ queue_name status offset_value limit_value job_class_name worker_id recurring_task_id ]
   attr_reader *PROPERTIES, :default_page_size
 
   delegate :last, :[], :reverse, to: :to_a
@@ -50,9 +50,15 @@ class ActiveJob::JobsRelation
   #   for large sets of jobs.
   # * <tt>:queue_name</tt> - To only include the jobs in the provided queue.
   # * <tt>:worker_id</tt> - To only include the jobs processed by the provided worker.
-  def where(job_class_name: nil, queue_name: nil, worker_id: nil)
+  # * <tt>:recurring_task_id</tt> - To only include the jobs corresponding to runs of a recurring task.
+  def where(job_class_name: nil, queue_name: nil, worker_id: nil, recurring_task_id: nil)
     # Remove nil arguments to avoid overriding parameters when concatenating +where+ clauses
-    arguments = { job_class_name: job_class_name, queue_name: queue_name, worker_id: worker_id }.compact.collect { |key, value| [ key, value.to_s ] }.to_h
+    arguments = { job_class_name: job_class_name,
+      queue_name: queue_name,
+      worker_id: worker_id,
+      recurring_task_id: recurring_task_id
+    }.compact.collect { |key, value| [ key, value.to_s ] }.to_h
+
     clone_with **arguments
   end
 
@@ -147,6 +153,11 @@ class ActiveJob::JobsRelation
     queue_adapter.discard_job(job, self)
   end
 
+  # Dispatch the provided job.
+  def dispatch_job(job)
+    queue_adapter.dispatch_job(job, self)
+  end
+
   # Find a job by id.
   #
   # Returns nil when not found.
@@ -258,7 +269,7 @@ class ActiveJob::JobsRelation
     end
 
     def filters
-      @filters ||= FILTERS.select { |property| public_send(property).present? && !queue_adapter.supports_filter?(self, property) }
+      @filters ||= FILTERS.select { |property| public_send(property).present? && !queue_adapter.supports_job_filter?(self, property) }
     end
 
     def ensure_failed_status

data/lib/active_job/queue_adapters/resque_ext.rb

@@ -47,7 +47,7 @@ module ActiveJob::QueueAdapters::ResqueExt
     ResquePauseHelper.paused?(queue_name)
   end
 
-  def supported_filters(jobs_relation)
+  def supported_job_filters(jobs_relation)
     if jobs_relation.pending? then [ :queue_name ]
     else []
     end
@@ -261,7 +261,7 @@ module ActiveJob::QueueAdapters::ResqueExt
           resque_job = job.raw_data
           resque_job["retried_at"] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
 
-          redis.lset(queue_redis_key, job.position, resque_job)
+          redis.lset(queue_redis_key, job.position, Resque.encode(resque_job))
           Resque::Job.create(resque_job["queue"], resque_job["payload"]["class"], *resque_job["payload"]["args"])
         rescue Redis::CommandError => error
           handle_resque_job_error(job, error)

data/lib/active_job/queue_adapters/solid_queue_ext/recurring_tasks.rb

@@ -0,0 +1,43 @@
+module ActiveJob::QueueAdapters::SolidQueueExt::RecurringTasks
+  def supports_recurring_tasks?
+    true
+  end
+
+  def recurring_tasks
+    tasks = recurring_tasks_from_dispatchers
+    last_enqueued_at_times = recurring_task_last_enqueued_at(tasks.keys)
+
+    recurring_tasks_from_dispatchers.collect do |task_id, task_attrs|
+      recurring_task_attributes_from_solid_queue_task_attributes(task_attrs).merge \
+        id: task_id,
+        last_enqueued_at: last_enqueued_at_times[task_id]
+    end
+  end
+
+  def find_recurring_task(task_id)
+    if task_attrs = recurring_tasks_from_dispatchers[task_id]
+      recurring_task_attributes_from_solid_queue_task_attributes(task_attrs).merge \
+        id: task_id,
+        last_enqueued_at: recurring_task_last_enqueued_at(task_id).values&.first
+    end
+  end
+
+  private
+    def recurring_tasks_from_dispatchers
+      SolidQueue::Process.where(kind: "Dispatcher").flat_map do |process|
+        process.metadata["recurring_schedule"]
+      end.compact.reduce({}, &:merge)
+    end
+
+    def recurring_task_attributes_from_solid_queue_task_attributes(task_attributes)
+      {
+        job_class_name: task_attributes["class_name"],
+        arguments: task_attributes["arguments"],
+        schedule: task_attributes["schedule"]
+      }
+    end
+
+    def recurring_task_last_enqueued_at(task_keys)
+      SolidQueue::RecurringExecution.where(task_key: task_keys).group(:task_key).maximum(:run_at)
+    end
+end

data/lib/active_job/queue_adapters/solid_queue_ext/workers.rb

@@ -0,0 +1,41 @@
+module ActiveJob::QueueAdapters::SolidQueueExt::Workers
+  def exposes_workers?
+    true
+  end
+
+  def fetch_workers(workers_relation)
+    solid_queue_processes_from_workers_relation(workers_relation).collect do |process|
+      worker_from_solid_queue_process(process)
+    end
+  end
+
+  def count_workers(workers_relation)
+    solid_queue_processes_from_workers_relation(workers_relation).count
+  end
+
+  def find_worker(worker_id)
+    if process = SolidQueue::Process.find_by(id: worker_id)
+      worker_attributes_from_solid_queue_process(process)
+    end
+  end
+
+  private
+    def solid_queue_processes_from_workers_relation(relation)
+      SolidQueue::Process.where(kind: "Worker").offset(relation.offset_value).limit(relation.limit_value)
+    end
+
+    def worker_from_solid_queue_process(process)
+      MissionControl::Jobs::Worker.new(queue_adapter: self, **worker_attributes_from_solid_queue_process(process))
+    end
+
+    def worker_attributes_from_solid_queue_process(process)
+      {
+        id: process.id,
+        name: "PID: #{process.pid}",
+        hostname: process.hostname,
+        last_heartbeat_at: process.last_heartbeat_at,
+        configuration: process.metadata,
+        raw_data: process.as_json
+      }
+    end
+end

data/lib/active_job/queue_adapters/solid_queue_ext.rb

@@ -1,5 +1,6 @@
 module ActiveJob::QueueAdapters::SolidQueueExt
   include MissionControl::Jobs::Adapter
+  include RecurringTasks, Workers
 
   def queues
     queues = SolidQueue::Queue.all
@@ -34,40 +35,26 @@ module ActiveJob::QueueAdapters::SolidQueueExt
     find_queue_by_name(queue_name).paused?
   end
 
-  def supported_statuses
-    RelationAdapter::STATUS_MAP.keys
+  def supported_job_statuses
+    SolidQueueJobs::STATUS_MAP.keys
   end
 
-  def supported_filters(*)
+  def supported_job_filters(*)
     [ :queue_name, :job_class_name ]
   end
 
-  def exposes_workers?
-    true
-  end
-
-  def workers
-    SolidQueue::Process.where(kind: "Worker").collect do |process|
-      worker_attributes_from_solid_queue_process(process)
-    end
-  end
-
-  def find_worker(worker_id)
-    if process = SolidQueue::Process.find_by(id: worker_id)
-      worker_attributes_from_solid_queue_process(process)
-    end
-  end
-
   def jobs_count(jobs_relation)
-    RelationAdapter.new(jobs_relation).count
+    SolidQueueJobs.new(jobs_relation).count
   end
 
   def fetch_jobs(jobs_relation)
-    find_solid_queue_jobs_within(jobs_relation).map { |job| deserialize_and_proxy_solid_queue_job(job, jobs_relation.status) }
+    SolidQueueJobs.new(jobs_relation).jobs.map do |job|
+      deserialize_and_proxy_solid_queue_job(job, jobs_relation.status)
+    end
   end
 
   def retry_all_jobs(jobs_relation)
-    RelationAdapter.new(jobs_relation).retry_all
+    SolidQueueJobs.new(jobs_relation).retry_all
   end
 
   def retry_job(job, jobs_relation)
@@ -75,13 +62,17 @@ module ActiveJob::QueueAdapters::SolidQueueExt
   end
 
   def discard_all_jobs(jobs_relation)
-    RelationAdapter.new(jobs_relation).discard_all
+    SolidQueueJobs.new(jobs_relation).discard_all
   end
 
   def discard_job(job, jobs_relation)
     find_solid_queue_job!(job.job_id, jobs_relation).discard
   end
 
+  def dispatch_job(job, jobs_relation)
+    dispatch_immediately find_solid_queue_job!(job.job_id, jobs_relation)
+  end
+
   def find_job(job_id, *)
     if job = SolidQueue::Job.where(active_job_id: job_id).order(:id).last
       deserialize_and_proxy_solid_queue_job job
@@ -93,27 +84,12 @@ module ActiveJob::QueueAdapters::SolidQueueExt
       SolidQueue::Queue.find_by_name(queue_name)
     end
 
-    def worker_attributes_from_solid_queue_process(process)
-      {
-        id: process.id,
-        name: "PID: #{process.pid}",
-        hostname: process.hostname,
-        last_heartbeat_at: process.last_heartbeat_at,
-        configuration: process.metadata,
-        raw_data: process.as_json
-      }
-    end
-
     def find_solid_queue_job!(job_id, jobs_relation)
       find_solid_queue_job(job_id, jobs_relation) or raise ActiveJob::Errors::JobNotFoundError.new(job_id, jobs_relation)
     end
 
     def find_solid_queue_job(job_id, jobs_relation)
-      RelationAdapter.new(jobs_relation).find_job(job_id)
-    end
-
-    def find_solid_queue_jobs_within(jobs_relation)
-      RelationAdapter.new(jobs_relation).jobs
+      SolidQueueJobs.new(jobs_relation).find_job(job_id)
     end
 
     def deserialize_and_proxy_solid_queue_job(solid_queue_job, job_status = nil)
@@ -134,7 +110,7 @@ module ActiveJob::QueueAdapters::SolidQueueExt
     end
 
     def status_from_solid_queue_job(solid_queue_job)
-      RelationAdapter::STATUS_MAP.invert[solid_queue_job.status]
+      SolidQueueJobs::STATUS_MAP.invert[solid_queue_job.status]
     end
 
     def execution_error_from_solid_queue_job(solid_queue_job)
@@ -146,7 +122,14 @@ module ActiveJob::QueueAdapters::SolidQueueExt
       end
     end
 
-    class RelationAdapter
+    def dispatch_immediately(job)
+      SolidQueue::Job.transaction do
+        job.dispatch_bypassing_concurrency_limits
+        job.blocked_execution.destroy!
+      end
+    end
+
+    class SolidQueueJobs
       STATUS_MAP = {
         pending: :ready,
         failed: :failed,
@@ -161,7 +144,7 @@ module ActiveJob::QueueAdapters::SolidQueueExt
       end
 
       def jobs
-        solid_queue_status.finished? ? order_finished_jobs(finished_jobs) : order_executions(executions).map(&:job)
+        solid_queue_status.finished? ? order_finished_jobs(finished_jobs) : order_executions(executions).map(&:job).compact
       end
 
       def count
@@ -169,7 +152,7 @@ module ActiveJob::QueueAdapters::SolidQueueExt
       end
 
       def find_job(active_job_id)
-        if job = SolidQueue::Job.find_by(active_job_id: active_job_id)
+        if job = SolidQueue::Job.where(active_job_id: active_job_id).order(:id).last
           job if matches_relation_filters?(job)
         end
       end
@@ -185,13 +168,16 @@ module ActiveJob::QueueAdapters::SolidQueueExt
       private
         attr_reader :jobs_relation
 
-        delegate :queue_name, :limit_value, :limit_value_provided?, :offset_value, :job_class_name, :default_page_size, :worker_id, to: :jobs_relation
+        delegate :queue_name, :limit_value, :limit_value_provided?, :offset_value, :job_class_name,
+          :default_page_size, :worker_id, :recurring_task_id, to: :jobs_relation
 
         def executions
-          execution_class_by_status.includes(job: "#{solid_queue_status}_execution")
+          execution_class_by_status
+            .then { |executions| include_execution_association(executions) }
             .then { |executions| filter_executions_by_queue(executions) }
             .then { |executions| filter_executions_by_class(executions) }
             .then { |executions| filter_executions_by_process_id(executions) }
+            .then { |executions| filter_executions_by_task_key(executions) }
             .then { |executions| limit(executions) }
             .then { |executions| offset(executions) }
         end
@@ -209,10 +195,11 @@ module ActiveJob::QueueAdapters::SolidQueueExt
         end
 
         def order_executions(executions)
-          # Follow polling order for scheduled executions, the rest by job_id
-          if solid_queue_status.scheduled? then executions.ordered
-          else
-            executions.order(:job_id)
+          case
+            # Follow polling order for scheduled executions, the rest by job_id, desc or asc
+          when solid_queue_status.scheduled? then executions.ordered
+          when recurring_task_id.present?    then executions.order(job_id: :desc)
+          else executions.order(job_id: :asc)
           end
         end
 
@@ -224,21 +211,26 @@ module ActiveJob::QueueAdapters::SolidQueueExt
           solid_queue_status.finished? ? finished_jobs.count : executions.count
         end
 
-        INTERNAL_COUNT_LIMIT = 500_000 # Hard limit to keep unlimited count queries fast enough
-
         def internally_limited_count
-          limited_count = solid_queue_status.finished? ? finished_jobs.limit(INTERNAL_COUNT_LIMIT + 1).count : executions.limit(INTERNAL_COUNT_LIMIT + 1).count
-          (limited_count == INTERNAL_COUNT_LIMIT + 1) ? Float::INFINITY : limited_count
+          count_limit = MissionControl::Jobs.internal_query_count_limit + 1
+          limited_count = solid_queue_status.finished? ? finished_jobs.limit(count_limit).count : executions.limit(count_limit).count
+          (limited_count == count_limit) ? Float::INFINITY : limited_count
         end
 
         def execution_class_by_status
-          if solid_queue_status.present? && !solid_queue_status.finished?
+          if recurring_task_id.present?
+            SolidQueue::RecurringExecution
+          elsif solid_queue_status.present? && !solid_queue_status.finished?
             "SolidQueue::#{solid_queue_status.capitalize}Execution".safe_constantize
           else
             raise ActiveJob::Errors::QueryError, "Status not supported: #{jobs_relation.status}"
           end
         end
 
+        def include_execution_association(executions)
+          solid_queue_status.present? ? executions.includes(job: "#{solid_queue_status}_execution") : executions.includes(:job)
+        end
+
         def filter_executions_by_queue(executions)
           return executions unless queue_name.present?
 
@@ -263,10 +255,14 @@ module ActiveJob::QueueAdapters::SolidQueueExt
           if solid_queue_status.claimed?
             executions.where(process_id: worker_id)
           else
-            raise ActiveJob::Errors::QueryError, "Filtering by worker ID is not supported for status #{jobs_relation.status}"
+            raise ActiveJob::Errors::QueryError, "Filtering by worker id is not supported for status #{jobs_relation.status}"
           end
         end
 
+        def filter_executions_by_task_key(executions)
+          recurring_task_id.present? ? executions.where(task_key: recurring_task_id) : executions
+        end
+
         def filter_jobs_by_class(jobs)
           job_class_name.present? ? jobs.where(class_name: job_class_name) : jobs
         end