solid_queue_guard 0.1.0 → 1.0.0

data/lib/generators/solid_queue_guard/install/ci_generator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'rails/generators/base'
+
+module SolidQueueGuard
+  module Generators
+    module Install
+      class CiGenerator < Rails::Generators::Base
+        source_root File.expand_path('templates', __dir__)
+
+        desc 'Adds a GitHub Actions workflow that runs solid_queue_guard:doctor in CI'
+
+        def copy_workflow
+          template 'solid_queue_guard.yml', '.github/workflows/solid_queue_guard.yml'
+        end
+      end
+    end
+  end
+end

data/lib/generators/solid_queue_guard/install/templates/solid_queue_guard.rb

@@ -11,7 +11,17 @@ SolidQueueGuard.configure do |config|
 
   config.failed_jobs_threshold = 20
   config.stale_process_threshold = 5.minutes
+  config.health_cache_ttl = 15.seconds
+  config.scheduled_backlog_threshold = 100
+
+  # config.disabled_checks = [:pidfile]
+  # config.checks.queue_lag = { threshold: 10.minutes }
+  # config.checks.failed_jobs = { threshold: 5, enabled: true }
+  # config.degraded_http_status = 207
+  # config.unhealthy_http_status = 503
 
   # config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
-  # config.notify_with = [ :rails_logger ]
+  # config.integrate_rails_health = true
+  # config.notify_with = [:rails_logger, :slack, :datadog, :webhook]
+  # config.metrics_backends = [:statsd, :prometheus, :opentelemetry]
 end

data/lib/generators/solid_queue_guard/install/templates/solid_queue_guard.yml

@@ -0,0 +1,25 @@
+name: Solid Queue Guard
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  doctor:
+    name: Solid Queue production readiness
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: .ruby-version
+          bundler-cache: true
+
+      - name: Prepare test database
+        run: bin/rails db:test:prepare
+
+      - name: Run Solid Queue Guard doctor
+        run: SOLID_QUEUE_GUARD_STRICT=1 bin/rails solid_queue_guard:doctor

data/lib/solid_queue_guard/checks/base.rb

@@ -8,6 +8,10 @@ module SolidQueueGuard
         new(**options).call
       end
 
+      def self.check_id
+        name.demodulize.underscore.delete_suffix('_check')
+      end
+
       def initialize(**options)
         @options = options
       end
@@ -47,6 +51,18 @@ module SolidQueueGuard
       def solid_queue_configuration
         @solid_queue_configuration ||= SolidQueue::Configuration.new
       end
+
+      def check_id
+        self.class.check_id
+      end
+
+      def guard_config
+        SolidQueueGuard.config
+      end
+
+      def check_setting(key, default = nil)
+        guard_config.check_setting(check_id, key, default)
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/config/async_supervisor_config_check.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Checks
+    module Config
+      class AsyncSupervisorConfigCheck < Base
+        def call
+          if PumaPluginSupport.async_supervisor_mode?
+            warn(
+              check_id,
+              'Solid Queue supervisor is running in async mode',
+              suggestion: [
+                'Review thread and database pool sizing;',
+                'the processes option in queue.yml is ignored in async mode'
+              ].join(' '),
+              metadata: { supervisor_mode: 'async' }
+            )
+          else
+            pass(check_id, 'Solid Queue supervisor is running in fork mode')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/checks/config/puma_colocated_check.rb

@@ -4,27 +4,21 @@ module SolidQueueGuard
   module Checks
     module Config
       class PumaColocatedCheck < Base
-        PUMA_PLUGIN_PATTERN = /plugin\s+:?solid_queue/
-
         def call
-          puma_config = rails_root.join('config/puma.rb')
-
-          return pass('puma_colocated', 'No config/puma.rb found') unless puma_config.exist?
-
-          content = puma_config.read
+          return pass(check_id, 'No config/puma.rb found') unless PumaPluginSupport.puma_config_path.exist?
 
-          if content.match?(PUMA_PLUGIN_PATTERN)
+          if PumaPluginSupport.puma_plugin_enabled?
             if Rails.env.production?
               warn(
-                'puma_colocated',
+                check_id,
                 'Solid Queue Puma plugin is enabled in production',
                 suggestion: 'Run Solid Queue in a dedicated job process for better isolation and memory management'
               )
             else
-              pass('puma_colocated', 'Solid Queue Puma plugin detected (non-production environment)')
+              pass(check_id, 'Solid Queue Puma plugin detected (non-production environment)')
             end
           else
-            pass('puma_colocated', 'Solid Queue is not co-located with Puma')
+            pass(check_id, 'Solid Queue is not co-located with Puma')
           end
         end
       end

data/lib/solid_queue_guard/checks/config/topology_recommendation_check.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Checks
+    module Config
+      class TopologyRecommendationCheck < Base
+        def call
+          recommendations = Recommendations::Topology.analyze
+
+          if recommendations.empty?
+            pass('topology_recommendation', 'No queue.yml topology changes recommended')
+          else
+            warn(
+              'topology_recommendation',
+              recommendations.join('; '),
+              suggestion: 'Review config/queue.yml worker and database pool settings',
+              metadata: { recommendations: recommendations }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/checks/registry.rb

@@ -14,7 +14,9 @@ module SolidQueueGuard
         Config::SchedulerConfigCheck,
         Config::EnvFlagsCheck,
         Config::ProcessHeartbeatConfigCheck,
-        Config::PumaColocatedCheck
+        Config::PumaColocatedCheck,
+        Config::TopologyRecommendationCheck,
+        Config::AsyncSupervisorConfigCheck
       ].freeze
 
       RUNTIME_CHECKS = [
@@ -29,10 +31,15 @@ module SolidQueueGuard
         Runtime::RecurringStaleCheck,
         Runtime::PausedQueueLagCheck,
         Runtime::PidfileCheck,
-        Runtime::FinishedJobsGrowthCheck
+        Runtime::FinishedJobsGrowthCheck,
+        Runtime::PumaPluginRuntimeCheck
       ].freeze
 
       class << self
+        def check_id_for(check_class)
+          check_class.check_id
+        end
+
         def for(scope)
           case scope.to_sym
           when :config then CONFIG_CHECKS

data/lib/solid_queue_guard/checks/runtime/base.rb

@@ -4,11 +4,7 @@ module SolidQueueGuard
   module Checks
     module Runtime
       class Base < Checks::Base
-        NOT_IMPLEMENTED_MESSAGE = 'Runtime check not implemented until v0.2'
-
-        def call
-          skip(check_id, NOT_IMPLEMENTED_MESSAGE)
-        end
+        include DatabaseSupport
 
         private
 

data/lib/solid_queue_guard/checks/runtime/blocked_jobs_check.rb

@@ -3,7 +3,26 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class BlockedJobsCheck < Base; end
+      class BlockedJobsCheck < Base
+        def call
+          with_queue_database do
+            blocked = SolidQueue::BlockedExecution.count
+            expired = SolidQueue::BlockedExecution.expired.count
+
+            if expired.positive?
+              warn(
+                check_id,
+                "#{expired} blocked job(s) have expired concurrency locks",
+                suggestion: 'Verify concurrency release maintenance is running'
+              )
+            elsif blocked.positive?
+              pass(check_id, "#{blocked} job(s) blocked by concurrency control")
+            else
+              pass(check_id, 'No blocked jobs')
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/database_support.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Checks
+    module Runtime
+      module DatabaseSupport
+        private
+
+        def queue_database_available?
+          pool = SolidQueue::Record.connection_pool
+          return false if pool.nil?
+
+          pool.with_connection(&:active?)
+        rescue StandardError
+          false
+        end
+
+        def with_queue_database
+          return skip(check_id, 'Queue database is not available') unless queue_database_available?
+
+          yield
+        rescue StandardError => e
+          skip(check_id, "Queue database query failed: #{e.class}")
+        end
+
+        def config
+          SolidQueueGuard.config
+        end
+
+        def lag_threshold_for(queue_name)
+          thresholds = config.check_setting(:queue_lag, :thresholds, config.queue_lag_thresholds)
+          per_queue = thresholds[queue_name.to_sym] || thresholds[queue_name]
+          return per_queue if per_queue
+
+          config.check_setting(:queue_lag, :threshold, thresholds[:default])
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/checks/runtime/dispatcher_check.rb

@@ -3,7 +3,32 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class DispatcherCheck < Base; end
+      class DispatcherCheck < Base
+        def call
+          with_queue_database do
+            dispatchers = SolidQueue::Process.where(kind: 'Dispatcher')
+            due_count = SolidQueue::ScheduledExecution.due.count
+
+            threshold = config.check_setting(:scheduled_backlog, :threshold, config.scheduled_backlog_threshold)
+
+            if dispatchers.none? && due_count.positive?
+              failure(
+                check_id,
+                "#{due_count} scheduled job(s) are due but no dispatcher is running",
+                suggestion: 'Start a dispatcher process or verify bin/jobs is running'
+              )
+            elsif due_count > threshold
+              warn(
+                check_id,
+                "#{due_count} scheduled executions are due (threshold: #{threshold})",
+                suggestion: 'Verify the dispatcher is keeping up with scheduled work'
+              )
+            else
+              pass(check_id, "Dispatcher healthy (#{due_count} due scheduled executions)")
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/failed_jobs_check.rb

@@ -3,7 +3,27 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class FailedJobsCheck < Base; end
+      class FailedJobsCheck < Base
+        WINDOW = 1.hour
+
+        def call
+          with_queue_database do
+            count = SolidQueue::FailedExecution.where(created_at: WINDOW.ago..).count
+            threshold = config.check_setting(:failed_jobs, :threshold, config.failed_jobs_threshold)
+
+            if count > threshold
+              warn(
+                check_id,
+                "#{count} failed job(s) in the last hour (threshold: #{threshold})",
+                suggestion: 'Review failed jobs in Mission Control or solid_queue_failed_executions',
+                metadata: { failed_jobs_last_hour: count }
+              )
+            else
+              pass(check_id, "#{count} failed job(s) in the last hour")
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/finished_jobs_growth_check.rb

@@ -3,7 +3,26 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class FinishedJobsGrowthCheck < Base; end
+      class FinishedJobsGrowthCheck < Base
+        WINDOW = 24.hours
+        GROWTH_THRESHOLD = 10_000
+
+        def call
+          with_queue_database do
+            recent_finished = SolidQueue::Job.where(finished_at: WINDOW.ago..).count
+
+            if recent_finished > GROWTH_THRESHOLD
+              warn(
+                check_id,
+                "#{recent_finished} finished jobs in the last 24 hours",
+                suggestion: 'Ensure a recurring cleanup task runs (SolidQueue::Job.clear_finished_in_batches)'
+              )
+            else
+              pass(check_id, "#{recent_finished} finished jobs in the last 24 hours")
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/orphaned_claims_check.rb

@@ -3,7 +3,24 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class OrphanedClaimsCheck < Base; end
+      class OrphanedClaimsCheck < Base
+        def call
+          with_queue_database do
+            alive_process_ids = SolidQueue::Process.pluck(:id)
+            orphaned = SolidQueue::ClaimedExecution.where.not(process_id: alive_process_ids)
+
+            if orphaned.none?
+              pass(check_id, 'No orphaned claimed executions')
+            else
+              warn(
+                check_id,
+                "#{orphaned.count} claimed execution(s) belong to dead processes",
+                suggestion: 'Restart workers; Solid Queue maintenance should release orphaned claims'
+              )
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/paused_queue_lag_check.rb

@@ -3,7 +3,28 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class PausedQueueLagCheck < Base; end
+      class PausedQueueLagCheck < Base
+        def call
+          with_queue_database do
+            paused_queues = SolidQueue::Pause.pluck(:queue_name)
+            return pass(check_id, 'No paused queues') if paused_queues.empty?
+
+            growing = paused_queues.select do |queue_name|
+              SolidQueue::ReadyExecution.exists?(queue_name: queue_name)
+            end
+
+            if growing.any?
+              warn(
+                check_id,
+                "Paused queue(s) still have ready jobs: #{growing.join(', ')}",
+                suggestion: 'Unpause queues or drain jobs before pausing'
+              )
+            else
+              pass(check_id, "Paused queues have no ready backlog: #{paused_queues.join(', ')}")
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/pidfile_check.rb

@@ -3,7 +3,39 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class PidfileCheck < Base; end
+      class PidfileCheck < Base
+        def call
+          pidfile = ENV['SOLID_QUEUE_PIDFILE'] || default_pidfile
+          return pass(check_id, 'No Solid Queue pidfile configured') if pidfile.blank?
+
+          path = Pathname(pidfile)
+          unless path.exist?
+            return warn(check_id, "Pidfile not found at #{path}",
+                        suggestion: 'Verify Solid Queue supervisor is running')
+          end
+
+          pid = path.read.strip.to_i
+          if process_alive?(pid)
+            pass(check_id, "Solid Queue supervisor pidfile present (pid #{pid})")
+          else
+            failure(check_id, "Pidfile exists but process #{pid} is not running",
+                    suggestion: 'Restart the Solid Queue supervisor')
+          end
+        end
+
+        private
+
+        def default_pidfile
+          rails_root.join('tmp/pids/solid_queue.pid').to_s
+        end
+
+        def process_alive?(pid)
+          Process.getpgid(pid)
+          true
+        rescue Errno::ESRCH
+          false
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/process_topology_check.rb

@@ -3,7 +3,37 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class ProcessTopologyCheck < Base; end
+      class ProcessTopologyCheck < Base
+        EXPECTED_KINDS = %w[Supervisor Worker Dispatcher Scheduler].freeze
+
+        def call
+          with_queue_database do
+            kinds = SolidQueue::Process.distinct.pluck(:kind)
+            missing = EXPECTED_KINDS - kinds
+
+            if missing == EXPECTED_KINDS
+              warn(check_id, 'No Solid Queue processes are running',
+                   suggestion: 'Start bin/jobs or the Solid Queue supervisor')
+            elsif missing.include?('Worker')
+              warn(check_id, 'No worker processes detected',
+                   suggestion: 'Ensure workers are configured in queue.yml and running')
+            elsif missing.include?('Dispatcher') && scheduled_work_expected?
+              warn(check_id, 'No dispatcher process detected with scheduled work configured')
+            else
+              pass(check_id, "Solid Queue processes running: #{kinds.sort.join(', ')}")
+            end
+          end
+        end
+
+        private
+
+        def scheduled_work_expected?
+          solid_queue_configuration.send(:recurring_tasks).any? ||
+            SolidQueue::ScheduledExecution.exists?
+        rescue StandardError
+          false
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/puma_plugin_runtime_check.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Checks
+    module Runtime
+      class PumaPluginRuntimeCheck < Base
+        def call
+          return skip(check_id, 'Solid Queue Puma plugin is not enabled') unless PumaPluginSupport.puma_plugin_enabled?
+
+          with_queue_database do
+            threshold = config.check_setting(:puma_plugin_runtime, :threshold, config.stale_process_threshold)
+            active = SolidQueue::Process.where(last_heartbeat_at: threshold.ago..)
+
+            if active.any?
+              pass(
+                check_id,
+                "Solid Queue processes are active via Puma plugin (#{active.count} process(es))",
+                metadata: { active_processes: active.count, supervisor_mode: supervisor_mode_label }
+              )
+            else
+              failure(
+                check_id,
+                'Solid Queue Puma plugin is enabled but no active processes were found',
+                suggestion: 'Verify the Puma plugin booted Solid Queue and workers are processing jobs',
+                metadata: { supervisor_mode: supervisor_mode_label }
+              )
+            end
+          end
+        end
+
+        private
+
+        def supervisor_mode_label
+          PumaPluginSupport.async_supervisor_mode? ? 'async' : 'fork'
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/checks/runtime/queue_lag_check.rb

@@ -3,7 +3,36 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class QueueLagCheck < Base; end
+      class QueueLagCheck < Base
+        def call
+          with_queue_database do
+            lags = SolidQueue::ReadyExecution.group(:queue_name).minimum(:created_at)
+            return pass(check_id, 'No jobs waiting in ready queues') if lags.empty?
+
+            worst = lags.map do |queue_name, oldest|
+              lag = Time.current - oldest
+              { queue: queue_name, lag: lag, threshold: lag_threshold_for(queue_name) }
+            end.max_by { |entry| entry[:lag] }
+
+            if worst[:lag] > worst[:threshold]
+              failure(
+                check_id,
+                "#{worst[:queue]} queue lag is #{worst[:lag].to_i} seconds",
+                suggestion: "Investigate worker coverage and throughput for the #{worst[:queue]} queue",
+                metadata: { queue_lag_seconds: worst[:lag].to_i, queue_name: worst[:queue] }
+              )
+            elsif worst[:lag] > worst[:threshold] / 2
+              warn(
+                check_id,
+                "#{worst[:queue]} queue lag is #{worst[:lag].to_i} seconds",
+                metadata: { queue_lag_seconds: worst[:lag].to_i, queue_name: worst[:queue] }
+              )
+            else
+              pass(check_id, "Queue lag within thresholds (worst: #{worst[:queue]} at #{worst[:lag].to_i}s)")
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/recurring_stale_check.rb

@@ -3,7 +3,35 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class RecurringStaleCheck < Base; end
+      class RecurringStaleCheck < Base
+        def call
+          with_queue_database do
+            tasks = SolidQueue::RecurringTask.static
+            return pass(check_id, 'No recurring tasks configured') if tasks.none?
+
+            stale_tasks = tasks.select do |task|
+              last_run = SolidQueue::RecurringExecution.where(task_key: task.key).maximum(:run_at)
+              last_run.nil? || last_run < expected_staleness.ago
+            end
+
+            if stale_tasks.any?
+              warn(
+                check_id,
+                "#{stale_tasks.size} recurring task(s) may be stale: #{stale_tasks.map(&:key).join(', ')}",
+                suggestion: 'Verify the scheduler process is running'
+              )
+            else
+              pass(check_id, 'Recurring tasks have recent executions')
+            end
+          end
+        end
+
+        private
+
+        def expected_staleness
+          config.check_setting(:recurring_stale, :threshold, config.stale_process_threshold * 2)
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/checks/runtime/scheduled_backlog_check.rb

@@ -3,7 +3,24 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class ScheduledBacklogCheck < Base; end
+      class ScheduledBacklogCheck < Base
+        def call
+          with_queue_database do
+            due_count = SolidQueue::ScheduledExecution.due.count
+            threshold = config.check_setting(:scheduled_backlog, :threshold, config.scheduled_backlog_threshold)
+
+            if due_count > threshold
+              warn(
+                check_id,
+                "Scheduled backlog is #{due_count} (threshold: #{threshold})",
+                suggestion: 'Check dispatcher health and scheduled job volume'
+              )
+            else
+              pass(check_id, "Scheduled backlog is #{due_count}")
+            end
+          end
+        end
+      end
     end
   end
 end