solid_queue 0.2.2 → 0.3.1

data/lib/solid_queue/processes/registrable.rb

@@ -5,28 +5,22 @@ module SolidQueue::Processes
     extend ActiveSupport::Concern
 
     included do
-      set_callback :boot, :after, :register
-      set_callback :boot, :after, :launch_heartbeat
+      after_boot :register, :launch_heartbeat
 
-      set_callback :shutdown, :before, :stop_heartbeat
-      set_callback :shutdown, :after, :deregister
+      before_shutdown :stop_heartbeat
+      after_shutdown :deregister
     end
 
-    def inspect
-      "#{kind}(pid=#{process_pid}, hostname=#{hostname}, metadata=#{metadata})"
-    end
-    alias to_s inspect
-
     private
       attr_accessor :process
 
       def register
         @process = SolidQueue::Process.register \
           kind: self.class.name.demodulize,
-          pid: process_pid,
+          pid: pid,
           hostname: hostname,
           supervisor: try(:supervisor),
-          metadata: metadata
+          metadata: metadata.compact
       end
 
       def deregister
@@ -38,7 +32,10 @@ module SolidQueue::Processes
       end
 
       def launch_heartbeat
-        @heartbeat_task = Concurrent::TimerTask.new(execution_interval: SolidQueue.process_heartbeat_interval) { heartbeat }
+        @heartbeat_task = Concurrent::TimerTask.new(execution_interval: SolidQueue.process_heartbeat_interval) do
+          wrap_in_app_executor { heartbeat }
+        end
+
         @heartbeat_task.execute
       end
 
@@ -49,21 +46,5 @@ module SolidQueue::Processes
       def heartbeat
         process.heartbeat
       end
-
-      def kind
-        self.class.name.demodulize
-      end
-
-      def hostname
-        @hostname ||= Socket.gethostname.force_encoding(Encoding::UTF_8)
-      end
-
-      def process_pid
-        @pid ||= ::Process.pid
-      end
-
-      def metadata
-        {}
-      end
   end
 end

data/lib/solid_queue/processes/runnable.rb

@@ -9,10 +9,11 @@ module SolidQueue::Processes
     def start
       @stopping = false
 
-      observe_initial_delay
-      run_callbacks(:boot) { boot }
+      SolidQueue.instrument(:start_process, process: self) do
+        run_callbacks(:boot) { boot }
+      end
 
-      start_loop
+      run
     end
 
     def stop
@@ -20,60 +21,45 @@ module SolidQueue::Processes
       @thread&.join
     end
 
-  private
-    DEFAULT_MODE = :async
-
-    def mode
-      (@mode || DEFAULT_MODE).to_s.inquiry
-    end
-
-    def boot
-      register_signal_handlers if supervised?
-      SolidQueue.logger.info("[SolidQueue] Starting #{self}")
-    end
+    private
+      DEFAULT_MODE = :async
 
-    def start_loop
-      if mode.async?
-        @thread = Thread.new { do_start_loop }
-      else
-        do_start_loop
+      def mode
+        (@mode || DEFAULT_MODE).to_s.inquiry
       end
-    end
 
-    def do_start_loop
-      loop do
-        break if shutting_down?
-
-        wrap_in_app_executor do
-          run
+      def boot
+        if supervised?
+          register_signal_handlers
+          set_procline
         end
       end
-    ensure
-      run_callbacks(:shutdown) { shutdown }
-    end
 
-    def shutting_down?
-      stopping? || supervisor_went_away? || finished?
-    end
+      def shutting_down?
+        stopping? || supervisor_went_away? || finished?
+      end
 
-    def run
-      raise NotImplementedError
-    end
+      def run
+        raise NotImplementedError
+      end
 
-    def stopping?
-      @stopping
-    end
+      def stopping?
+        @stopping
+      end
 
-    def finished?
-      running_inline? && all_work_completed?
-    end
+      def finished?
+        running_inline? && all_work_completed?
+      end
 
-    def all_work_completed?
-      false
-    end
+      def all_work_completed?
+        false
+      end
 
-    def running_inline?
-      mode.inline?
-    end
+      def set_procline
+      end
+
+      def running_inline?
+        mode.inline?
+      end
   end
 end

data/lib/solid_queue/processes/signals.rb

@@ -38,7 +38,7 @@ module SolidQueue::Processes
         when :QUIT
           request_immediate_termination
         else
-          SolidQueue.logger.warn "Received unhandled signal #{signal}"
+          SolidQueue.instrument :unhandled_signal_error, signal: signal
         end
       end
 

data/lib/solid_queue/processes/supervised.rb

@@ -14,6 +14,10 @@ module SolidQueue::Processes
     end
 
     private
+      def set_procline
+        procline "waiting"
+      end
+
       def supervisor_went_away?
         supervised? && supervisor&.pid != ::Process.ppid
       end

data/lib/solid_queue/supervisor.rb

@@ -4,8 +4,6 @@ module SolidQueue
   class Supervisor < Processes::Base
     include Processes::Signals
 
-    set_callback :boot, :after, :launch_process_prune
-
     class << self
       def start(mode: :work, load_configuration_from: nil)
         SolidQueue.supervisor = true
@@ -23,6 +21,8 @@ module SolidQueue
     def start
       run_callbacks(:boot) { boot }
 
+      start_forks
+      launch_process_prune
       supervise
     rescue Processes::GracefulTerminationRequested
       graceful_termination
@@ -42,8 +42,6 @@ module SolidQueue
       end
 
       def supervise
-        start_forks
-
         loop do
           procline "supervising #{forks.keys.join(", ")}"
 
@@ -63,15 +61,15 @@ module SolidQueue
         end
       end
 
+      def start_forks
+        configured_processes.each { |configured_process| start_fork(configured_process) }
+      end
+
       def launch_process_prune
         @prune_task = Concurrent::TimerTask.new(run_now: true, execution_interval: SolidQueue.process_alive_threshold) { prune_dead_processes }
         @prune_task.execute
       end
 
-      def start_forks
-        configured_processes.each { |configured_process| start_fork(configured_process) }
-      end
-
       def shutdown
         stop_process_prune
         restore_default_signal_handlers
@@ -79,19 +77,24 @@ module SolidQueue
       end
 
       def graceful_termination
-        SolidQueue.logger.info("[SolidQueue] Terminating gracefully...")
-        term_forks
+        SolidQueue.instrument(:graceful_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do |payload|
+          term_forks
 
-        wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
-          reap_terminated_forks
-        end
+          wait_until(SolidQueue.shutdown_timeout, -> { all_forks_terminated? }) do
+            reap_terminated_forks
+          end
 
-        immediate_termination unless all_forks_terminated?
+          unless all_forks_terminated?
+            payload[:shutdown_timeout_exceeded] = true
+            immediate_termination
+          end
+        end
       end
 
       def immediate_termination
-        SolidQueue.logger.info("[SolidQueue] Terminating immediately...")
-        quit_forks
+        SolidQueue.instrument(:immediate_termination, supervisor_pid: ::Process.pid, supervised_pids: forks.keys) do
+          quit_forks
+        end
       end
 
       def term_forks
@@ -111,9 +114,7 @@ module SolidQueue
       end
 
       def prune_dead_processes
-        wrap_in_app_executor do
-          SolidQueue::Process.prune
-        end
+        wrap_in_app_executor { SolidQueue::Process.prune }
       end
 
       def start_fork(configured_process)
@@ -147,11 +148,11 @@ module SolidQueue
       end
 
       def replace_fork(pid, status)
-        if supervised_fork = forks.delete(pid)
-          SolidQueue.logger.info "[SolidQueue] Restarting fork[#{status.pid}] (status: #{status.exitstatus})"
-          start_fork(supervised_fork)
-        else
-          SolidQueue.logger.info "[SolidQueue] Tried to replace fork[#{pid}] (status: #{status.exitstatus}, fork[#{status.pid}]), but it had already died  (status: #{status.exitstatus})"
+        SolidQueue.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
+          if supervised_fork = forks.delete(pid)
+            payload[:fork] = supervised_fork
+            start_fork(supervised_fork)
+          end
         end
       end
 

data/lib/solid_queue/version.rb

@@ -1,3 +1,3 @@
 module SolidQueue
-  VERSION = "0.2.2"
+  VERSION = "0.3.1"
 end

data/lib/solid_queue/worker.rb

@@ -2,7 +2,7 @@
 
 module SolidQueue
   class Worker < Processes::Base
-    include Processes::Runnable, Processes::Poller
+    include Processes::Poller
 
     attr_accessor :queues, :pool
 
@@ -14,41 +14,40 @@ module SolidQueue
       @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
     end
 
-    private
-      def run
-        polled_executions = poll
-
-        if polled_executions.size > 0
-          procline "performing #{polled_executions.count} jobs"
+    def metadata
+      super.merge(queues: queues.join(","), thread_pool_size: pool.size)
+    end
 
-          polled_executions.each do |execution|
+    private
+      def poll
+        claim_executions.then do |executions|
+          executions.each do |execution|
             pool.post(execution)
           end
-        else
-          procline "waiting for jobs in #{queues.join(",")}"
-          interruptible_sleep(polling_interval)
+
+          executions.size
         end
       end
 
-      def poll
+      def claim_executions
         with_polling_volume do
           SolidQueue::ReadyExecution.claim(queues, pool.idle_threads, process.id)
         end
       end
 
       def shutdown
-        super
-
         pool.shutdown
         pool.wait_for_termination(SolidQueue.shutdown_timeout)
+
+        super
       end
 
       def all_work_completed?
         SolidQueue::ReadyExecution.aggregated_count_across(queues).zero?
       end
 
-      def metadata
-        super.merge(queues: queues.join(","), thread_pool_size: pool.size)
+      def set_procline
+        procline "waiting for jobs in #{queues.join(",")}"
       end
   end
 end

data/lib/solid_queue.rb

@@ -3,26 +3,23 @@
 require "solid_queue/version"
 require "solid_queue/engine"
 
-require "active_job/queue_adapters/solid_queue_adapter"
-require "active_job/concurrency_controls"
-
-require "solid_queue/app_executor"
-require "solid_queue/processes/supervised"
-require "solid_queue/processes/registrable"
-require "solid_queue/processes/interruptible"
-require "solid_queue/processes/pidfile"
-require "solid_queue/processes/procline"
-require "solid_queue/processes/poller"
-require "solid_queue/processes/base"
-require "solid_queue/processes/runnable"
-require "solid_queue/processes/signals"
-require "solid_queue/configuration"
-require "solid_queue/pool"
-require "solid_queue/worker"
-require "solid_queue/dispatcher"
-require "solid_queue/supervisor"
+require "active_job"
+require "active_job/queue_adapters"
+
+require "active_support"
+require "active_support/core_ext/numeric/time"
+
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false)
+loader.ignore("#{__dir__}/solid_queue/tasks.rb")
+loader.ignore("#{__dir__}/generators")
+loader.ignore("#{__dir__}/puma")
+loader.setup
 
 module SolidQueue
+  extend self
+
   mattr_accessor :logger, default: ActiveSupport::Logger.new($stdout)
   mattr_accessor :app_executor, :on_thread_error, :connects_to
 
@@ -33,6 +30,8 @@ module SolidQueue
 
   mattr_accessor :shutdown_timeout, default: 5.seconds
 
+  mattr_accessor :enqueue_after_transaction_commit, default: false
+
   mattr_accessor :silence_polling, default: true
 
   mattr_accessor :supervisor_pidfile
@@ -42,11 +41,19 @@ module SolidQueue
   mattr_accessor :clear_finished_jobs_after, default: 1.day
   mattr_accessor :default_concurrency_control_period, default: 3.minutes
 
-  def self.supervisor?
+  def supervisor?
     supervisor
   end
 
-  def self.silence_polling?
+  def silence_polling?
     silence_polling
   end
+
+  def preserve_finished_jobs?
+    preserve_finished_jobs
+  end
+
+  def instrument(channel, **options, &block)
+    ActiveSupport::Notifications.instrument("#{channel}.solid_queue", **options, &block)
+  end
 end

metadata

@@ -1,29 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.1
 platform: ruby
 authors:
 - Rosa Gutierrez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-05 00:00:00.000000000 Z
+date: 2024-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rails
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+- !ruby/object:Gem::Dependency
+  name: fugit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.1
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +122,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails-omakase
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Database-backed Active Job backend.
 email:
 - rosa@37signals.com
@@ -86,6 +198,7 @@ files:
 - app/models/solid_queue/job/clearable.rb
 - app/models/solid_queue/job/concurrency_controls.rb
 - app/models/solid_queue/job/executable.rb
+- app/models/solid_queue/job/recurrable.rb
 - app/models/solid_queue/job/schedulable.rb
 - app/models/solid_queue/pause.rb
 - app/models/solid_queue/process.rb
@@ -94,11 +207,13 @@ files:
 - app/models/solid_queue/queue_selector.rb
 - app/models/solid_queue/ready_execution.rb
 - app/models/solid_queue/record.rb
+- app/models/solid_queue/recurring_execution.rb
 - app/models/solid_queue/scheduled_execution.rb
 - app/models/solid_queue/semaphore.rb
 - config/routes.rb
 - db/migrate/20231211200639_create_solid_queue_tables.rb
 - db/migrate/20240110143450_add_missing_index_to_blocked_executions.rb
+- db/migrate/20240218110712_create_recurring_executions.rb
 - lib/active_job/concurrency_controls.rb
 - lib/active_job/queue_adapters/solid_queue_adapter.rb
 - lib/generators/solid_queue/install/USAGE
@@ -109,9 +224,14 @@ files:
 - lib/solid_queue/app_executor.rb
 - lib/solid_queue/configuration.rb
 - lib/solid_queue/dispatcher.rb
+- lib/solid_queue/dispatcher/concurrency_maintenance.rb
+- lib/solid_queue/dispatcher/recurring_schedule.rb
+- lib/solid_queue/dispatcher/recurring_task.rb
 - lib/solid_queue/engine.rb
+- lib/solid_queue/log_subscriber.rb
 - lib/solid_queue/pool.rb
 - lib/solid_queue/processes/base.rb
+- lib/solid_queue/processes/callbacks.rb
 - lib/solid_queue/processes/interruptible.rb
 - lib/solid_queue/processes/pidfile.rb
 - lib/solid_queue/processes/poller.rb
@@ -124,12 +244,12 @@ files:
 - lib/solid_queue/tasks.rb
 - lib/solid_queue/version.rb
 - lib/solid_queue/worker.rb
-homepage: https://github.com/basecamp/solid_queue
+homepage: https://github.com/rails/solid_queue
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/basecamp/solid_queue
-  source_code_uri: https://github.com/basecamp/solid_queue
+  homepage_uri: https://github.com/rails/solid_queue
+  source_code_uri: https://github.com/rails/solid_queue
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -145,7 +265,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: Database-backed Active Job backend.