solid_queue 0.2.2 → 0.3.1

data/lib/solid_queue/configuration.rb

@@ -4,7 +4,7 @@ module SolidQueue
   class Configuration
     WORKER_DEFAULTS = {
       queues: "*",
-      threads: 5,
+      threads: 3,
       processes: 1,
       polling_interval: 0.1
     }
@@ -12,7 +12,9 @@ module SolidQueue
     DISPATCHER_DEFAULTS = {
       batch_size: 500,
       polling_interval: 1,
-      concurrency_maintenance_interval: 600
+      concurrency_maintenance: true,
+      concurrency_maintenance_interval: 600,
+      recurring_tasks: []
     }
 
     def initialize(mode: :work, load_from: nil)
@@ -33,7 +35,7 @@ module SolidQueue
       if mode.in? %i[ work all]
         workers_options.flat_map do |worker_options|
           processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
-          processes.times.collect { SolidQueue::Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
+          processes.times.map { Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
         end
       else
         []
@@ -42,8 +44,10 @@ module SolidQueue
 
     def dispatchers
       if mode.in? %i[ dispatch all]
-        dispatchers_options.flat_map do |dispatcher_options|
-          SolidQueue::Dispatcher.new(**dispatcher_options)
+        dispatchers_options.map do |dispatcher_options|
+          recurring_tasks = parse_recurring_tasks dispatcher_options[:recurring_tasks]
+
+          Dispatcher.new **dispatcher_options.merge(recurring_tasks: recurring_tasks).with_defaults(DISPATCHER_DEFAULTS)
         end
       end
     end
@@ -73,6 +77,11 @@ module SolidQueue
           .map { |options| options.dup.symbolize_keys }
       end
 
+      def parse_recurring_tasks(tasks)
+        Array(tasks).map do |id, options|
+          Dispatcher::RecurringTask.from_configuration(id, **options)
+        end.select(&:valid?)
+      end
 
       def load_config_from(file_or_hash)
         case file_or_hash

data/lib/solid_queue/dispatcher/concurrency_maintenance.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Dispatcher::ConcurrencyMaintenance
+    include AppExecutor
+
+    attr_reader :interval, :batch_size
+
+    def initialize(interval, batch_size)
+      @interval = interval
+      @batch_size = batch_size
+    end
+
+    def start
+      @concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: interval) do
+        expire_semaphores
+        unblock_blocked_executions
+      end
+
+      @concurrency_maintenance_task.add_observer do |_, _, error|
+        handle_thread_error(error) if error
+      end
+
+      @concurrency_maintenance_task.execute
+    end
+
+    def stop
+      @concurrency_maintenance_task.shutdown
+    end
+
+    private
+      def expire_semaphores
+        wrap_in_app_executor do
+          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
+        end
+      end
+
+      def unblock_blocked_executions
+        wrap_in_app_executor do
+          BlockedExecution.unblock(batch_size)
+        end
+      end
+  end
+end

data/lib/solid_queue/dispatcher/recurring_schedule.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  class Dispatcher::RecurringSchedule
+    include AppExecutor
+
+    attr_reader :configured_tasks, :scheduled_tasks
+
+    def initialize(tasks)
+      @configured_tasks = Array(tasks).map { |task| Dispatcher::RecurringTask.wrap(task) }
+      @scheduled_tasks = Concurrent::Hash.new
+    end
+
+    def load_tasks
+      configured_tasks.each do |task|
+        load_task(task)
+      end
+    end
+
+    def load_task(task)
+      scheduled_tasks[task.key] = schedule(task)
+    end
+
+    def unload_tasks
+      scheduled_tasks.values.each(&:cancel)
+      scheduled_tasks.clear
+    end
+
+    def tasks
+      configured_tasks.each_with_object({}) { |task, hsh| hsh[task.key] = task.to_h }
+    end
+
+    def inspect
+      configured_tasks.map(&:to_s).join(" | ")
+    end
+
+    private
+      def schedule(task)
+        scheduled_task = Concurrent::ScheduledTask.new(task.delay_from_now, args: [ self, task, task.next_time ]) do |thread_schedule, thread_task, thread_task_run_at|
+          thread_schedule.load_task(thread_task)
+
+          wrap_in_app_executor do
+            thread_task.enqueue(at: thread_task_run_at)
+          end
+        end
+
+        scheduled_task.add_observer do |_, _, error|
+          # Don't notify on task cancellation before execution, as this will happen normally
+          # as part of unloading tasks
+          handle_thread_error(error) if error && !error.is_a?(Concurrent::CancelledOperationError)
+        end
+
+        scheduled_task.tap(&:execute)
+      end
+  end
+end

data/lib/solid_queue/dispatcher/recurring_task.rb

@@ -0,0 +1,91 @@
+require "fugit"
+
+module SolidQueue
+  class Dispatcher::RecurringTask
+    class << self
+      def wrap(args)
+        args.is_a?(self) ? args : from_configuration(args.first, **args.second)
+      end
+
+      def from_configuration(key, **options)
+        new(key, class_name: options[:class], schedule: options[:schedule], arguments: options[:args])
+      end
+    end
+
+    attr_reader :key, :schedule, :class_name, :arguments
+
+    def initialize(key, class_name:, schedule:, arguments: nil)
+      @key = key
+      @class_name = class_name
+      @schedule = schedule
+      @arguments = Array(arguments)
+    end
+
+    def delay_from_now
+      [ (next_time - Time.current).to_f, 0 ].max
+    end
+
+    def next_time
+      parsed_schedule.next_time.utc
+    end
+
+    def enqueue(at:)
+      SolidQueue.instrument(:enqueue_recurring_task, task: key, at: at) do |payload|
+        if using_solid_queue_adapter?
+          perform_later_and_record(run_at: at)
+        else
+          payload[:other_adapter] = true
+
+          perform_later
+        end.tap do |active_job|
+          payload[:active_job_id] = active_job&.job_id
+        end
+      end
+    end
+
+    def valid?
+      parsed_schedule.instance_of?(Fugit::Cron)
+    end
+
+    def to_s
+      "#{class_name}.perform_later(#{arguments.map(&:inspect).join(",")}) [ #{parsed_schedule.original} ]"
+    end
+
+    def to_h
+      {
+        schedule: schedule,
+        class_name: class_name,
+        arguments: arguments
+      }
+    end
+
+    private
+      def using_solid_queue_adapter?
+        job_class.queue_adapter_name.inquiry.solid_queue?
+      end
+
+      def perform_later_and_record(run_at:)
+        RecurringExecution.record(key, run_at) { perform_later }
+      end
+
+      def perform_later
+        job_class.perform_later(*arguments_with_kwargs)
+      end
+
+      def arguments_with_kwargs
+        if arguments.last.is_a?(Hash)
+          arguments[0...-1] + [ Hash.ruby2_keywords_hash(arguments.last) ]
+        else
+          arguments
+        end
+      end
+
+      def parsed_schedule
+        @parsed_schedule ||= Fugit.parse(schedule)
+      end
+
+      def job_class
+        @job_class ||= class_name.safe_constantize
+      end
+  end
+end

data/lib/solid_queue/dispatcher.rb

@@ -2,72 +2,57 @@
 
 module SolidQueue
   class Dispatcher < Processes::Base
-    include Processes::Runnable, Processes::Poller
+    include Processes::Poller
 
-    attr_accessor :batch_size, :concurrency_maintenance_interval
+    attr_accessor :batch_size, :concurrency_maintenance, :recurring_schedule
 
-    set_callback :boot, :after, :launch_concurrency_maintenance
-    set_callback :shutdown, :before, :stop_concurrency_maintenance
+    after_boot :start_concurrency_maintenance, :load_recurring_schedule
+    before_shutdown :stop_concurrency_maintenance, :unload_recurring_schedule
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
 
       @batch_size = options[:batch_size]
       @polling_interval = options[:polling_interval]
-      @concurrency_maintenance_interval = options[:concurrency_maintenance_interval]
+
+      @concurrency_maintenance = ConcurrencyMaintenance.new(options[:concurrency_maintenance_interval], options[:batch_size]) if options[:concurrency_maintenance]
+      @recurring_schedule = RecurringSchedule.new(options[:recurring_tasks])
+    end
+
+    def metadata
+      super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.tasks.presence)
     end
 
     private
-      def run
+      def poll
         batch = dispatch_next_batch
-
-        unless batch.size > 0
-          procline "waiting"
-          interruptible_sleep(polling_interval)
-        end
+        batch.size
       end
 
       def dispatch_next_batch
         with_polling_volume do
-          SolidQueue::ScheduledExecution.dispatch_next_batch(batch_size)
+          ScheduledExecution.dispatch_next_batch(batch_size)
         end
       end
 
-      def launch_concurrency_maintenance
-        @concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: concurrency_maintenance_interval) do
-          expire_semaphores
-          unblock_blocked_executions
-        end
-
-        @concurrency_maintenance_task.add_observer do |_, _, error|
-          handle_thread_error(error) if error
-        end
-
-        @concurrency_maintenance_task.execute
+      def start_concurrency_maintenance
+        concurrency_maintenance&.start
       end
 
-      def stop_concurrency_maintenance
-        @concurrency_maintenance_task.shutdown
+      def load_recurring_schedule
+        recurring_schedule.load_tasks
       end
 
-      def expire_semaphores
-        wrap_in_app_executor do
-          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
-        end
-      end
-
-      def unblock_blocked_executions
-        wrap_in_app_executor do
-          BlockedExecution.unblock(batch_size)
-        end
+      def stop_concurrency_maintenance
+        concurrency_maintenance&.stop
       end
 
-      def initial_jitter
-        Kernel.rand(0...polling_interval)
+      def unload_recurring_schedule
+        recurring_schedule.unload_tasks
       end
 
-      def metadata
-        super.merge(batch_size: batch_size)
+      def set_procline
+        procline "waiting"
       end
   end
 end

data/lib/solid_queue/engine.rb

@@ -18,7 +18,7 @@ module SolidQueue
 
     initializer "solid_queue.app_executor", before: :run_prepare_callbacks do |app|
       config.solid_queue.app_executor    ||= app.executor
-      config.solid_queue.on_thread_error ||= -> (exception) { Rails.error.report(exception, handled: false) }
+      config.solid_queue.on_thread_error ||= ->(exception) { Rails.error.report(exception, handled: false) }
 
       SolidQueue.app_executor = config.solid_queue.app_executor
       SolidQueue.on_thread_error = config.solid_queue.on_thread_error
@@ -26,8 +26,10 @@ module SolidQueue
 
     initializer "solid_queue.logger" do |app|
       ActiveSupport.on_load(:solid_queue) do
-        self.logger = app.logger
+        self.logger ||= app.logger
       end
+
+      SolidQueue::LogSubscriber.attach_to :solid_queue
     end
 
     initializer "solid_queue.active_job.extensions" do

data/lib/solid_queue/log_subscriber.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "active_support/log_subscriber"
+
+class SolidQueue::LogSubscriber < ActiveSupport::LogSubscriber
+  def dispatch_scheduled(event)
+    debug formatted_event(event, action: "Dispatch scheduled jobs", **event.payload.slice(:batch_size, :size))
+  end
+
+  def release_many_claimed(event)
+    debug formatted_event(event, action: "Release claimed jobs", **event.payload.slice(:size))
+  end
+
+  def release_claimed(event)
+    debug formatted_event(event, action: "Release claimed job", **event.payload.slice(:job_id, :process_id))
+  end
+
+  def retry_all(event)
+    debug formatted_event(event, action: "Retry failed jobs", **event.payload.slice(:jobs_size, :size))
+  end
+
+  def retry(event)
+    debug formatted_event(event, action: "Retry failed job", **event.payload.slice(:job_id))
+  end
+
+  def discard_all(event)
+    debug formatted_event(event, action: "Discard jobs", **event.payload.slice(:jobs_size, :size, :status))
+  end
+
+  def discard(event)
+    debug formatted_event(event, action: "Discard job", **event.payload.slice(:job_id, :status))
+  end
+
+  def release_many_blocked(event)
+    debug formatted_event(event, action: "Unblock jobs", **event.payload.slice(:limit, :size))
+  end
+
+  def release_blocked(event)
+    debug formatted_event(event, action: "Release blocked job", **event.payload.slice(:job_id, :concurrency_key, :released))
+  end
+
+  def enqueue_recurring_task(event)
+    attributes = event.payload.slice(:task, :at, :active_job_id)
+
+    if event.payload[:other_adapter]
+      debug formatted_event(event, action: "Enqueued recurring task outside Solid Queue", **attributes)
+    else
+      action = attributes[:active_job_id].present? ? "Enqueued recurring task" : "Skipped recurring task – already dispatched"
+      info formatted_event(event, action: action, **attributes)
+    end
+  end
+
+  def start_process(event)
+    process = event.payload[:process]
+
+    attributes = {
+      pid: process.pid,
+      hostname: process.hostname
+    }.merge(process.metadata)
+
+    info formatted_event(event, action: "Started #{process.kind}", **attributes)
+  end
+
+  def shutdown_process(event)
+    process = event.payload[:process]
+
+    attributes = {
+      pid: process.pid,
+      hostname: process.hostname
+    }.merge(process.metadata)
+
+    info formatted_event(event, action: "Shut down #{process.kind}", **attributes)
+  end
+
+  def register_process(event)
+    process_kind = event.payload[:kind]
+    attributes = event.payload.slice(:pid, :hostname)
+
+    if error = event.payload[:error]
+      warn formatted_event(event, action: "Error registering #{process_kind}", **attributes.merge(error: formatted_error(error)))
+    else
+      info formatted_event(event, action: "Register #{process_kind}", **attributes)
+    end
+  end
+
+  def deregister_process(event)
+    process = event.payload[:process]
+
+    attributes = {
+      process_id: process.id,
+      pid: process.pid,
+      hostname: process.hostname,
+      last_heartbeat_at: process.last_heartbeat_at,
+      claimed_size: process.claimed_executions.size,
+      pruned: event.payload
+    }
+
+    if error = event.payload[:error]
+      warn formatted_event(event, action: "Error deregistering #{process.kind}", **attributes.merge(error: formatted_error(error)))
+    else
+      info formatted_event(event, action: "Deregister #{process.kind}", **attributes)
+    end
+  end
+
+  def prune_processes(event)
+    debug formatted_event(event, action: "Prune dead processes", **event.payload.slice(:size))
+  end
+
+  def thread_error(event)
+    error formatted_event(event, action: "Error in thread", error: formatted_error(event.payload[:error]))
+  end
+
+  def graceful_termination(event)
+    attributes = event.payload.slice(:supervisor_pid, :supervised_pids)
+
+    if event.payload[:shutdown_timeout_exceeded]
+      warn formatted_event(event, action: "Supervisor wasn't terminated gracefully - shutdown timeout exceeded", **attributes)
+    else
+      info formatted_event(event, action: "Supervisor terminated gracefully", **attributes)
+    end
+  end
+
+  def immediate_termination(event)
+    info formatted_event(event, action: "Supervisor terminated immediately", **event.payload.slice(:supervisor_pid, :supervised_pids))
+  end
+
+  def unhandled_signal_error(event)
+    error formatted_event(event, action: "Received unhandled signal", **event.payload.slice(:signal))
+  end
+
+  def replace_fork(event)
+    status = event.payload[:status]
+    attributes = event.payload.slice(:pid).merge \
+      status: (status.exitstatus || "no exit status set"),
+      pid_from_status: status.pid,
+      signaled: status.signaled?,
+      stopsig: status.stopsig,
+      termsig: status.termsig
+
+    if replaced_fork = event.payload[:fork]
+      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname))
+    else
+      warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes)
+    end
+  end
+
+  private
+    def formatted_event(event, action:, **attributes)
+      "SolidQueue-#{SolidQueue::VERSION} #{action} (#{event.duration.round(1)}ms)  #{formatted_attributes(**attributes)}"
+    end
+
+    def formatted_attributes(**attributes)
+      attributes.map { |attr, value| "#{attr}: #{value.inspect}" }.join(", ")
+    end
+
+    def formatted_error(error)
+      [ error.class, error.message ].compact.join(" ")
+    end
+
+    # Use the logger configured for SolidQueue
+    def logger
+      SolidQueue.logger
+    end
+end

data/lib/solid_queue/processes/base.rb

@@ -3,25 +3,24 @@
 module SolidQueue
   module Processes
     class Base
-      include ActiveSupport::Callbacks
-      define_callbacks :boot, :shutdown
-
+      include Callbacks # Defines callbacks needed by other concerns
       include AppExecutor, Registrable, Interruptible, Procline
 
-      private
-        def observe_initial_delay
-          interruptible_sleep(initial_jitter)
-        end
+      def kind
+        self.class.name.demodulize
+      end
 
-        def boot
-        end
+      def hostname
+        @hostname ||= Socket.gethostname.force_encoding(Encoding::UTF_8)
+      end
 
-        def shutdown
-        end
+      def pid
+        @pid ||= ::Process.pid
+      end
 
-        def initial_jitter
-          0
-        end
+      def metadata
+        {}
+      end
     end
   end
 end

data/lib/solid_queue/processes/callbacks.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Callbacks
+    extend ActiveSupport::Concern
+
+    included do
+      extend ActiveModel::Callbacks
+      define_model_callbacks :boot, :shutdown
+    end
+
+    private
+      def boot
+      end
+
+      def shutdown
+      end
+  end
+end

data/lib/solid_queue/processes/interruptible.rb

@@ -10,7 +10,7 @@ module SolidQueue::Processes
       SELF_PIPE_BLOCK_SIZE = 11
 
       def interrupt
-        self_pipe[:writer].write_nonblock( "." )
+        self_pipe[:writer].write_nonblock(".")
       rescue Errno::EAGAIN, Errno::EINTR
         # Ignore writes that would block and retry
         # if another signal arrived while writing

data/lib/solid_queue/processes/poller.rb

@@ -4,11 +4,45 @@ module SolidQueue::Processes
   module Poller
     extend ActiveSupport::Concern
 
+    include Runnable
+
     included do
       attr_accessor :polling_interval
     end
 
+    def metadata
+      super.merge(polling_interval: polling_interval)
+    end
+
     private
+      def run
+        if mode.async?
+          @thread = Thread.new { start_loop }
+        else
+          start_loop
+        end
+      end
+
+      def start_loop
+        loop do
+          break if shutting_down?
+
+          wrap_in_app_executor do
+            unless poll > 0
+              interruptible_sleep(polling_interval)
+            end
+          end
+        end
+      ensure
+        SolidQueue.instrument(:shutdown_process, process: self) do
+          run_callbacks(:shutdown) { shutdown }
+        end
+      end
+
+      def poll
+        raise NotImplementedError
+      end
+
       def with_polling_volume
         if SolidQueue.silence_polling?
           ActiveRecord::Base.logger.silence { yield }
@@ -16,9 +50,5 @@ module SolidQueue::Processes
           yield
         end
       end
-
-      def metadata
-        super.merge(polling_interval: polling_interval)
-      end
   end
 end