solid_queue 0.2.1 → 0.3.0

data/lib/solid_queue/dispatcher.rb

@@ -2,72 +2,57 @@
 
 module SolidQueue
   class Dispatcher < Processes::Base
-    include Processes::Runnable, Processes::Poller
+    include Processes::Poller
 
-    attr_accessor :batch_size, :concurrency_maintenance_interval
+    attr_accessor :batch_size, :concurrency_maintenance, :recurring_schedule
 
-    set_callback :boot, :after, :launch_concurrency_maintenance
-    set_callback :shutdown, :before, :stop_concurrency_maintenance
+    after_boot :start_concurrency_maintenance, :load_recurring_schedule
+    before_shutdown :stop_concurrency_maintenance, :unload_recurring_schedule
 
     def initialize(**options)
       options = options.dup.with_defaults(SolidQueue::Configuration::DISPATCHER_DEFAULTS)
 
       @batch_size = options[:batch_size]
       @polling_interval = options[:polling_interval]
-      @concurrency_maintenance_interval = options[:concurrency_maintenance_interval]
+
+      @concurrency_maintenance = ConcurrencyMaintenance.new(options[:concurrency_maintenance_interval], options[:batch_size]) if options[:concurrency_maintenance]
+      @recurring_schedule = RecurringSchedule.new(options[:recurring_tasks])
     end
 
     private
-      def run
+      def poll
         batch = dispatch_next_batch
-
-        unless batch.size > 0
-          procline "waiting"
-          interruptible_sleep(polling_interval)
-        end
+        batch.size
       end
 
       def dispatch_next_batch
         with_polling_volume do
-          SolidQueue::ScheduledExecution.dispatch_next_batch(batch_size)
+          ScheduledExecution.dispatch_next_batch(batch_size)
         end
       end
 
-      def launch_concurrency_maintenance
-        @concurrency_maintenance_task = Concurrent::TimerTask.new(run_now: true, execution_interval: concurrency_maintenance_interval) do
-          expire_semaphores
-          unblock_blocked_executions
-        end
-
-        @concurrency_maintenance_task.add_observer do |_, _, error|
-          handle_thread_error(error) if error
-        end
-
-        @concurrency_maintenance_task.execute
+      def start_concurrency_maintenance
+        concurrency_maintenance&.start
       end
 
-      def stop_concurrency_maintenance
-        @concurrency_maintenance_task.shutdown
+      def load_recurring_schedule
+        recurring_schedule.load_tasks
       end
 
-      def expire_semaphores
-        wrap_in_app_executor do
-          Semaphore.expired.in_batches(of: batch_size, &:delete_all)
-        end
+      def stop_concurrency_maintenance
+        concurrency_maintenance&.stop
       end
 
-      def unblock_blocked_executions
-        wrap_in_app_executor do
-          BlockedExecution.unblock(batch_size)
-        end
+      def unload_recurring_schedule
+        recurring_schedule.unload_tasks
       end
 
-      def initial_jitter
-        Kernel.rand(0...polling_interval)
+      def set_procline
+        procline "waiting"
       end
 
       def metadata
-        super.merge(batch_size: batch_size)
+        super.merge(batch_size: batch_size, concurrency_maintenance_interval: concurrency_maintenance&.interval, recurring_schedule: recurring_schedule.tasks.presence )
       end
   end
 end

data/lib/solid_queue/processes/base.rb

@@ -3,25 +3,8 @@
 module SolidQueue
   module Processes
     class Base
-      include ActiveSupport::Callbacks
-      define_callbacks :boot, :shutdown
-
+      include Callbacks # Defines callbacks needed by other concerns
       include AppExecutor, Registrable, Interruptible, Procline
-
-      private
-        def observe_initial_delay
-          interruptible_sleep(initial_jitter)
-        end
-
-        def boot
-        end
-
-        def shutdown
-        end
-
-        def initial_jitter
-          0
-        end
     end
   end
 end

data/lib/solid_queue/processes/callbacks.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SolidQueue::Processes
+  module Callbacks
+    extend ActiveSupport::Concern
+
+    included do
+      extend ActiveModel::Callbacks
+      define_model_callbacks :boot, :shutdown
+    end
+
+    private
+      def boot
+      end
+
+      def shutdown
+      end
+  end
+end

data/lib/solid_queue/processes/poller.rb

@@ -4,11 +4,39 @@ module SolidQueue::Processes
   module Poller
     extend ActiveSupport::Concern
 
+    include Runnable
+
     included do
       attr_accessor :polling_interval
     end
 
     private
+      def run
+        if mode.async?
+          @thread = Thread.new { start_loop }
+        else
+          start_loop
+        end
+      end
+
+      def start_loop
+        loop do
+          break if shutting_down?
+
+          wrap_in_app_executor do
+            unless poll > 0
+              interruptible_sleep(polling_interval)
+            end
+          end
+        end
+      ensure
+        run_callbacks(:shutdown) { shutdown }
+      end
+
+      def poll
+        raise NotImplementedError
+      end
+
       def with_polling_volume
         if SolidQueue.silence_polling?
           ActiveRecord::Base.logger.silence { yield }

data/lib/solid_queue/processes/registrable.rb

@@ -5,11 +5,10 @@ module SolidQueue::Processes
     extend ActiveSupport::Concern
 
     included do
-      set_callback :boot, :after, :register
-      set_callback :boot, :after, :launch_heartbeat
+      after_boot :register, :launch_heartbeat
 
-      set_callback :shutdown, :before, :stop_heartbeat
-      set_callback :shutdown, :after, :deregister
+      before_shutdown :stop_heartbeat
+      after_shutdown :deregister
     end
 
     def inspect
@@ -26,7 +25,7 @@ module SolidQueue::Processes
           pid: process_pid,
           hostname: hostname,
           supervisor: try(:supervisor),
-          metadata: metadata
+          metadata: metadata.compact
       end
 
       def deregister
@@ -55,7 +54,7 @@ module SolidQueue::Processes
       end
 
       def hostname
-        @hostname ||= Socket.gethostname
+        @hostname ||= Socket.gethostname.force_encoding(Encoding::UTF_8)
       end
 
       def process_pid

data/lib/solid_queue/processes/runnable.rb

@@ -8,11 +8,9 @@ module SolidQueue::Processes
 
     def start
       @stopping = false
-
-      observe_initial_delay
       run_callbacks(:boot) { boot }
 
-      start_loop
+      run
     end
 
     def stop
@@ -20,60 +18,47 @@ module SolidQueue::Processes
       @thread&.join
     end
 
-  private
-    DEFAULT_MODE = :async
+    private
+      DEFAULT_MODE = :async
 
-    def mode
-      (@mode || DEFAULT_MODE).to_s.inquiry
-    end
+      def mode
+        (@mode || DEFAULT_MODE).to_s.inquiry
+      end
 
-    def boot
-      register_signal_handlers if supervised?
-      SolidQueue.logger.info("[SolidQueue] Starting #{self}")
-    end
+      def boot
+        if supervised?
+          register_signal_handlers
+          set_procline
+        end
 
-    def start_loop
-      if mode.async?
-        @thread = Thread.new { do_start_loop }
-      else
-        do_start_loop
+        SolidQueue.logger.info("[SolidQueue] Starting #{self}")
       end
-    end
 
-    def do_start_loop
-      loop do
-        break if shutting_down?
-
-        wrap_in_app_executor do
-          run
-        end
+      def shutting_down?
+        stopping? || supervisor_went_away? || finished?
       end
-    ensure
-      run_callbacks(:shutdown) { shutdown }
-    end
 
-    def shutting_down?
-      stopping? || supervisor_went_away? || finished?
-    end
+      def run
+        raise NotImplementedError
+      end
 
-    def run
-      raise NotImplementedError
-    end
+      def stopping?
+        @stopping
+      end
 
-    def stopping?
-      @stopping
-    end
+      def finished?
+        running_inline? && all_work_completed?
+      end
 
-    def finished?
-      running_inline? && all_work_completed?
-    end
+      def all_work_completed?
+        false
+      end
 
-    def all_work_completed?
-      false
-    end
+      def set_procline
+      end
 
-    def running_inline?
-      mode.inline?
-    end
+      def running_inline?
+        mode.inline?
+      end
   end
 end

data/lib/solid_queue/processes/supervised.rb

@@ -14,6 +14,10 @@ module SolidQueue::Processes
     end
 
     private
+      def set_procline
+        procline "waiting"
+      end
+
       def supervisor_went_away?
         supervised? && supervisor&.pid != ::Process.ppid
       end

data/lib/solid_queue/recurring_tasks/manager.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module RecurringTasks
+    class Manager < Processes::Base
+      include Processes::Runnable
+
+      attr_accessor :schedule
+
+      after_boot :load_schedule
+      before_shutdown :unload_schedule
+
+      def initialize(tasks)
+        @schedule = Schedule.new(tasks)
+      end
+
+      private
+        def load_schedule
+          schedule.load_tasks
+        end
+
+        def unload_schedule
+          schedule.unload_tasks
+        end
+
+        def metadata
+          super.merge(schedule: schedule.tasks)
+        end
+    end
+  end
+end

data/lib/solid_queue/recurring_tasks/schedule.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SolidQueue
+  module RecurringTasks
+    class Schedule
+      include AppExecutor
+
+      attr_reader :configured_tasks, :scheduled_tasks
+
+      def initialize(tasks)
+        @configured_tasks = Array(tasks).map { |task| Task.wrap(task) }
+        @scheduled_tasks = Concurrent::Hash.new
+      end
+
+      def load_tasks
+        configured_tasks.each do |task|
+          load_task(task)
+        end
+      end
+
+      def load_task(task)
+        scheduled_tasks[task.key] = schedule(task)
+      end
+
+      def unload_tasks
+        scheduled_tasks.values.each(&:cancel)
+        scheduled_tasks.clear
+      end
+
+      def tasks
+        configured_tasks.each_with_object({}) { |task, hsh| hsh[task.key] = task.to_h }
+      end
+
+      def inspect
+        configured_tasks.map(&:to_s).join(" | ")
+      end
+
+      private
+        def schedule(task)
+          scheduled_task = Concurrent::ScheduledTask.new(task.delay_from_now, args: [ self, task, task.next_time ]) do |thread_schedule, thread_task, thread_task_run_at|
+            thread_schedule.load_task(thread_task)
+
+            wrap_in_app_executor do
+              thread_task.enqueue(at: thread_task_run_at)
+            end
+          end
+
+          scheduled_task.add_observer do |_, _, error|
+            # Don't notify on task cancellation before execution, as this will happen normally
+            # as part of unloading tasks
+            handle_thread_error(error) if error && !error.is_a?(Concurrent::CancelledOperationError)
+          end
+
+          scheduled_task.tap(&:execute)
+        end
+    end
+  end
+end

data/lib/solid_queue/recurring_tasks/task.rb

@@ -0,0 +1,87 @@
+require "fugit"
+
+module SolidQueue
+  module RecurringTasks
+    class Task
+      class << self
+        def wrap(args)
+          args.is_a?(self) ? args : from_configuration(args.first, **args.second)
+        end
+
+        def from_configuration(key, **options)
+          new(key, class_name: options[:class], schedule: options[:schedule], arguments: options[:args])
+        end
+      end
+
+      attr_reader :key, :schedule, :class_name, :arguments
+
+      def initialize(key, class_name:, schedule:, arguments: nil)
+        @key = key
+        @class_name = class_name
+        @schedule = schedule
+        @arguments = Array(arguments)
+      end
+
+      def delay_from_now
+        [ (next_time - Time.current).to_f, 0 ].max
+      end
+
+      def next_time
+        parsed_schedule.next_time.utc
+      end
+
+      def enqueue(at:)
+        if using_solid_queue_adapter?
+          perform_later_and_record(run_at: at)
+        else
+          perform_later
+        end
+      end
+
+      def valid?
+        parsed_schedule.instance_of?(Fugit::Cron)
+      end
+
+      def to_s
+        "#{class_name}.perform_later(#{arguments.map(&:inspect).join(",")}) [ #{parsed_schedule.original.to_s} ]"
+      end
+
+      def to_h
+        {
+          schedule: schedule,
+          class_name: class_name,
+          arguments: arguments
+        }
+      end
+
+      private
+        def using_solid_queue_adapter?
+          job_class.queue_adapter_name.inquiry.solid_queue?
+        end
+
+        def perform_later_and_record(run_at:)
+          RecurringExecution.record(key, run_at) { perform_later.provider_job_id }
+        end
+
+        def perform_later
+          job_class.perform_later(*arguments_with_kwargs)
+        end
+
+        def arguments_with_kwargs
+          if arguments.last.is_a?(Hash)
+            arguments[0...-1] + [ Hash.ruby2_keywords_hash(arguments.last) ]
+          else
+            arguments
+          end
+        end
+
+        def parsed_schedule
+          @parsed_schedule ||= Fugit.parse(schedule)
+        end
+
+        def job_class
+          @job_class ||= class_name.safe_constantize
+        end
+    end
+  end
+end

data/lib/solid_queue/supervisor.rb

@@ -4,7 +4,7 @@ module SolidQueue
   class Supervisor < Processes::Base
     include Processes::Signals
 
-    set_callback :boot, :after, :launch_process_prune
+    after_boot :launch_process_prune
 
     class << self
       def start(mode: :work, load_configuration_from: nil)

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/solid_queue/worker.rb

@@ -2,7 +2,7 @@
 
 module SolidQueue
   class Worker < Processes::Base
-    include Processes::Runnable, Processes::Poller
+    include Processes::Poller
 
     attr_accessor :queues, :pool
 
@@ -15,22 +15,17 @@ module SolidQueue
     end
 
     private
-      def run
-        polled_executions = poll
-
-        if polled_executions.size > 0
-          procline "performing #{polled_executions.count} jobs"
-
-          polled_executions.each do |execution|
+      def poll
+        claim_executions.then do |executions|
+          executions.each do |execution|
             pool.post(execution)
           end
-        else
-          procline "waiting for jobs in #{queues.join(",")}"
-          interruptible_sleep(polling_interval)
+
+          executions.size
         end
       end
 
-      def poll
+      def claim_executions
         with_polling_volume do
           SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process.id)
         end
@@ -47,6 +42,10 @@ module SolidQueue
         SolidQueue::ReadyExecution.aggregated_count_across(queues).zero?
       end
 
+      def set_procline
+        procline "waiting for jobs in #{queues.join(",")}"
+      end
+
       def metadata
         super.merge(queues: queues.join(","), thread_pool_size: pool.size)
       end

data/lib/solid_queue.rb

@@ -3,24 +3,16 @@
 require "solid_queue/version"
 require "solid_queue/engine"
 
-require "active_job/queue_adapters/solid_queue_adapter"
-require "active_job/concurrency_controls"
-
-require "solid_queue/app_executor"
-require "solid_queue/processes/supervised"
-require "solid_queue/processes/registrable"
-require "solid_queue/processes/interruptible"
-require "solid_queue/processes/pidfile"
-require "solid_queue/processes/procline"
-require "solid_queue/processes/poller"
-require "solid_queue/processes/base"
-require "solid_queue/processes/runnable"
-require "solid_queue/processes/signals"
-require "solid_queue/configuration"
-require "solid_queue/pool"
-require "solid_queue/worker"
-require "solid_queue/dispatcher"
-require "solid_queue/supervisor"
+require "active_job"
+require "active_job/queue_adapters"
+
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false)
+loader.ignore("#{__dir__}/solid_queue/tasks.rb")
+loader.ignore("#{__dir__}/generators")
+loader.ignore("#{__dir__}/puma")
+loader.setup
 
 module SolidQueue
   mattr_accessor :logger, default: ActiveSupport::Logger.new($stdout)
@@ -33,7 +25,7 @@ module SolidQueue
 
   mattr_accessor :shutdown_timeout, default: 5.seconds
 
-  mattr_accessor :silence_polling, default: false
+  mattr_accessor :silence_polling, default: true
 
   mattr_accessor :supervisor_pidfile
   mattr_accessor :supervisor, default: false
@@ -42,11 +34,17 @@ module SolidQueue
   mattr_accessor :clear_finished_jobs_after, default: 1.day
   mattr_accessor :default_concurrency_control_period, default: 3.minutes
 
-  def self.supervisor?
-    supervisor
-  end
+  class << self
+    def supervisor?
+      supervisor
+    end
+
+    def silence_polling?
+      silence_polling
+    end
 
-  def self.silence_polling?
-    silence_polling
+    def preserve_finished_jobs?
+      preserve_finished_jobs
+    end
   end
 end