solid_queue_guard 0.1.0 → 1.0.0

data/lib/solid_queue_guard/checks/runtime/stale_process_check.rb

@@ -3,7 +3,25 @@
 module SolidQueueGuard
   module Checks
     module Runtime
-      class StaleProcessCheck < Base; end
+      class StaleProcessCheck < Base
+        def call
+          with_queue_database do
+            threshold = config.check_setting(:stale_process, :threshold, config.stale_process_threshold)
+            stale = SolidQueue::Process.where(last_heartbeat_at: ...threshold.ago)
+
+            if stale.none?
+              pass(check_id, 'All Solid Queue processes have recent heartbeats')
+            else
+              failure(
+                check_id,
+                "#{stale.count} Solid Queue process(es) have stale heartbeats",
+                suggestion: 'Restart the Solid Queue supervisor and verify workers are running',
+                metadata: { stale_processes: stale.count }
+              )
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/solid_queue_guard/cli.rb

@@ -18,9 +18,12 @@ module SolidQueueGuard
       strict = strict_flag?(argv) if strict.nil?
 
       report = Runner.new(scope: scope).run
+      Notifier.deliver_all(report) if notify?(report)
+      Metrics::Exporter.export(report) if SolidQueueGuard.config.metrics_backends.any?
+
       output = formatter_for(format).new(report).render
 
-      Rails.logger.debug output unless output.empty?
+      $stdout.puts(output) unless output.empty?
 
       exit report.exit_code(strict: strict)
     end
@@ -51,5 +54,11 @@ module SolidQueueGuard
         ENV.fetch('SOLID_QUEUE_GUARD_SCOPE', 'config').to_sym
       end
     end
+
+    def notify?(report)
+      return false unless SolidQueueGuard.config.notify_with.any?
+
+      report.status != :healthy
+    end
   end
 end

data/lib/solid_queue_guard/configuration.rb

@@ -11,7 +11,12 @@ module SolidQueueGuard
                   :health_cache_ttl,
                   :scheduled_backlog_threshold,
                   :integrate_rails_health,
-                  :notify_with
+                  :notify_with,
+                  :metrics_backends,
+                  :disabled_checks,
+                  :checks,
+                  :degraded_http_status,
+                  :unhealthy_http_status
 
     def initialize
       @enabled = true
@@ -24,10 +29,40 @@ module SolidQueueGuard
       @scheduled_backlog_threshold = 100
       @integrate_rails_health = false
       @notify_with = [:rails_logger]
+      @metrics_backends = []
+      @disabled_checks = []
+      @checks = ActiveSupport::OrderedOptions.new
+      @degraded_http_status = :ok
+      @unhealthy_http_status = :service_unavailable
     end
 
     def strict?
       strict_mode || ActiveModel::Type::Boolean.new.cast(ENV.fetch('SOLID_QUEUE_GUARD_STRICT', nil))
     end
+
+    def check_enabled?(check_id)
+      id = check_id.to_sym
+      return false if disabled_checks.map(&:to_sym).include?(id)
+
+      settings = check_settings_for(id)
+      return true if settings.nil?
+
+      enabled_value = settings[:enabled]
+      enabled_value = settings['enabled'] if enabled_value.nil?
+      enabled_value != false
+    end
+
+    def check_settings_for(check_id)
+      checks[check_id.to_sym] || checks[check_id.to_s]
+    end
+
+    def check_setting(check_id, key, default = nil)
+      settings = check_settings_for(check_id)
+      return default unless settings
+
+      value = settings[key]
+      value = settings[key.to_sym] if value.nil?
+      value.nil? ? default : value
+    end
   end
 end

data/lib/solid_queue_guard/engine.rb

@@ -17,5 +17,17 @@ module SolidQueueGuard
         end
       end
     end
+
+    initializer 'solid_queue_guard.rails_health' do
+      next unless SolidQueueGuard.config.integrate_rails_health
+
+      ActiveSupport.on_load(:action_controller_base) do
+        Rails::HealthController.class_eval do
+          def solid_queue_guard_status
+            SolidQueueGuard::Health::Cache.fetch[:status]
+          end
+        end
+      end
+    end
   end
 end

data/lib/solid_queue_guard/formatters/terminal.rb

@@ -27,7 +27,7 @@ module SolidQueueGuard
       attr_reader :report
 
       def check_lines
-        report.results.map do |result|
+        report.results.reject(&:skip?).map do |result|
           icon = ICONS.fetch(result.status, '•')
           "#{icon} #{result.message}"
         end

data/lib/solid_queue_guard/health/cache.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Health
+    class Cache
+      def self.fetch
+        new.fetch
+      end
+
+      def fetch
+        Rails.cache.fetch(cache_key, expires_in: SolidQueueGuard.config.health_cache_ttl) do
+          build_payload
+        end
+      end
+
+      private
+
+      def cache_key
+        'solid_queue_guard/health'
+      end
+
+      def build_payload
+        report = Runner.new(scope: :all).run
+        Payload.new(report).to_h
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/health/payload.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Health
+    class Payload
+      def initialize(report)
+        @report = report
+      end
+
+      def to_h
+        base = report.to_h.merge(
+          queue_lag_seconds: queue_lag_seconds,
+          failed_jobs_last_hour: failed_jobs_last_hour,
+          dead_processes: dead_processes
+        )
+
+        base[:recommendations] = topology_recommendations if topology_recommendations.any?
+        base
+      end
+
+      private
+
+      attr_reader :report
+
+      def queue_lag_seconds
+        metadata_value('queue_lag') { |metadata| metadata[:queue_lag_seconds] }
+      end
+
+      def failed_jobs_last_hour
+        metadata_value('failed_jobs') { |metadata| metadata[:failed_jobs_last_hour] } || 0
+      end
+
+      def dead_processes
+        metadata_value('stale_process') { |metadata| metadata[:stale_processes] } || 0
+      end
+
+      def metadata_value(check_id)
+        result = report.results.find { |entry| entry.id == check_id }
+        metadata = result&.metadata || {}
+        value = yield(metadata)
+        value unless value.nil?
+      end
+
+      def topology_recommendations
+        result = report.results.find { |entry| entry.id == 'topology_recommendation' }
+        result&.metadata&.fetch(:recommendations, []) || []
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/http_status_policy.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  # @api private
+  module HttpStatusPolicy
+    module_function
+
+    STATUS_MAP = {
+      ok: :ok,
+      success: :ok,
+      200 => :ok,
+      multi_status: :multi_status,
+      207 => :multi_status,
+      service_unavailable: :service_unavailable,
+      unavailable: :service_unavailable,
+      503 => :service_unavailable
+    }.freeze
+
+    def resolve(status)
+      key = status.is_a?(Integer) ? status : status.to_sym
+      STATUS_MAP.fetch(key, :ok)
+    end
+
+    def for_report_status(report_status, config: SolidQueueGuard.config)
+      case report_status.to_s
+      when 'unhealthy'
+        resolve(config.unhealthy_http_status)
+      when 'degraded'
+        resolve(config.degraded_http_status)
+      else
+        :ok
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/metrics/exporter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Metrics
+    class Exporter
+      STATUS_VALUES = { healthy: 0, degraded: 1, unhealthy: 2 }.freeze
+
+      def self.export(report, backends: SolidQueueGuard.config.metrics_backends)
+        new(report, backends: backends).export
+      end
+
+      def initialize(report, backends:)
+        @report = report
+        @backends = Array(backends)
+      end
+
+      def export
+        backends.each { |backend| export_backend(backend) }
+      end
+
+      private
+
+      attr_reader :report, :backends
+
+      def export_backend(backend)
+        case backend.to_sym
+        when :statsd then Statsd.export(report)
+        when :prometheus then Prometheus.export(report)
+        when :opentelemetry then OpenTelemetry.export(report)
+        end
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/metrics/open_telemetry.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Metrics
+    module OpenTelemetry
+      module_function
+
+      def export(report)
+        return unless defined?(::OpenTelemetry)
+
+        meter = ::OpenTelemetry.meter_provider.meter('solid_queue_guard')
+        gauge = meter.create_gauge('solid_queue.guard.overall_status', unit: 'status')
+        gauge.record(Exporter::STATUS_VALUES.fetch(report.status))
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/metrics/prometheus.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Metrics
+    module Prometheus
+      module_function
+
+      def export(report)
+        path = ENV.fetch('SOLID_QUEUE_GUARD_PROMETHEUS_FILE', Rails.root.join('tmp/solid_queue_guard.prom'))
+        File.write(path, render(report))
+      end
+
+      def render(report)
+        <<~PROM
+          # TYPE solid_queue_guard_overall_status gauge
+          solid_queue_guard_overall_status #{Exporter::STATUS_VALUES.fetch(report.status)}
+        PROM
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/metrics/statsd.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Metrics
+    module Statsd
+      module_function
+
+      def export(report)
+        require 'socket'
+        host = ENV.fetch('SOLID_QUEUE_GUARD_STATSD_HOST', '127.0.0.1')
+        port = ENV.fetch('SOLID_QUEUE_GUARD_STATSD_PORT', 8125).to_i
+        socket = UDPSocket.new
+        socket.send(metric_line(report), 0, host, port)
+      ensure
+        socket&.close
+      end
+
+      def metric_line(report)
+        "solid_queue.guard.overall_status:#{STATUS_VALUES.fetch(report.status)}|g"
+      end
+
+      STATUS_VALUES = Exporter::STATUS_VALUES
+    end
+  end
+end

data/lib/solid_queue_guard/notifier.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  class Notifier
+    class << self
+      def deliver_all(report, adapters: SolidQueueGuard.config.notify_with)
+        Array(adapters).each do |adapter|
+          deliver(adapter, report)
+        end
+      end
+
+      def deliver(adapter, report)
+        case adapter.to_sym
+        when :slack then deliver_slack(report)
+        when :datadog then deliver_datadog(report)
+        when :webhook then deliver_webhook(report)
+        else deliver_rails_logger(report)
+        end
+      end
+
+      def post_json(url, payload, headers: {})
+        require 'net/http'
+        uri = URI(url)
+        request = Net::HTTP::Post.new(uri)
+        request['Content-Type'] = 'application/json'
+        headers.each { |key, value| request[key] = value }
+        request.body = JSON.generate(payload)
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+          http.request(request)
+        end
+      end
+
+      private
+
+      def deliver_rails_logger(report)
+        Rails.logger.warn("[SolidQueueGuard] status=#{report.status} warnings=#{report.warnings.join('; ')}")
+      end
+
+      def deliver_slack(report)
+        webhook_url = ENV.fetch('SOLID_QUEUE_GUARD_SLACK_WEBHOOK_URL', nil)
+        return if webhook_url.blank?
+
+        post_json(webhook_url, { text: "SolidQueueGuard status: *#{report.status}*\n#{report.warnings.join("\n")}" })
+      end
+
+      def deliver_datadog(report)
+        api_key = ENV.fetch('DD_API_KEY', nil)
+        return if api_key.blank?
+
+        alert_type = case report.status
+                     when :unhealthy then 'error'
+                     when :degraded then 'warning'
+                     else 'info'
+                     end
+
+        post_json(
+          'https://api.datadoghq.com/api/v1/events',
+          {
+            title: 'SolidQueueGuard alert',
+            text: report.warnings.join("\n"),
+            alert_type: alert_type,
+            tags: ["status:#{report.status}"]
+          },
+          headers: { 'DD-API-KEY' => api_key }
+        )
+      end
+
+      def deliver_webhook(report)
+        webhook_url = ENV.fetch('SOLID_QUEUE_GUARD_WEBHOOK_URL', nil)
+        return if webhook_url.blank?
+
+        post_json(webhook_url, report.to_h)
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/notifiers/base.rb

@@ -4,14 +4,9 @@ module SolidQueueGuard
   module Notifiers
     # @api private
     class Base
-      def self.deliver(_report)
-        raise NotImplementedError, 'Notification adapters ship in v0.3'
+      def self.deliver(report)
+        Notifier.deliver(:rails_logger, report)
       end
     end
-
-    class RailsLogger < Base; end
-    class Slack < Base; end
-    class Datadog < Base; end
-    class Webhook < Base; end
   end
 end

data/lib/solid_queue_guard/puma_plugin_support.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  # @api private
+  module PumaPluginSupport
+    module_function
+
+    PUMA_PLUGIN_PATTERN = /plugin\s+:?solid_queue/
+    ASYNC_MODE_PATTERN = /solid_queue_mode\s+:?async/
+
+    def puma_config_path
+      Rails.root.join('config/puma.rb')
+    end
+
+    def puma_config_content
+      return unless puma_config_path.exist?
+
+      puma_config_path.read
+    end
+
+    def puma_plugin_enabled?
+      content = puma_config_content
+      content.present? && content.match?(PUMA_PLUGIN_PATTERN)
+    end
+
+    def puma_async_mode?
+      content = puma_config_content
+      content.present? && content.match?(ASYNC_MODE_PATTERN)
+    end
+
+    def async_supervisor_mode?
+      ENV.fetch('SOLID_QUEUE_SUPERVISOR_MODE', 'fork').casecmp('async').zero? || puma_async_mode?
+    end
+  end
+end

data/lib/solid_queue_guard/recommendations/topology.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module SolidQueueGuard
+  module Recommendations
+    class Topology
+      def self.analyze(configuration: SolidQueue::Configuration.new)
+        new(configuration: configuration).analyze
+      end
+
+      def initialize(configuration:)
+        @configuration = configuration
+      end
+
+      def analyze
+        recommendations = []
+        recommendations.concat(worker_recommendations)
+        recommendations.concat(pool_recommendations)
+        recommendations.uniq
+      end
+
+      private
+
+      attr_reader :configuration
+
+      def worker_recommendations
+        worker_queues = QueueCoverage.worker_queues_from_configuration(configuration)
+        uncovered = uncovered_queues(worker_queues)
+        return [] if uncovered.empty?
+
+        ["Add worker coverage for: #{uncovered.join(', ')}"]
+      end
+
+      def pool_recommendations
+        return async_pool_recommendation if PumaPluginSupport.async_supervisor_mode?
+
+        required_threads = configuration.estimated_number_of_threads
+        pool_size = SolidQueue::Record.connection_pool&.size
+        return [] if pool_size.nil? || required_threads <= pool_size
+
+        ["Increase queue DB pool to at least #{required_threads + 2} (currently #{pool_size})"]
+      rescue StandardError
+        []
+      end
+
+      def uncovered_queues(assigned_queues)
+        observed = observed_queues
+        (observed - assigned_queues).sort
+      end
+
+      def observed_queues
+        return [] unless database_available?
+
+        SolidQueue::Job.distinct.pluck(:queue_name)
+      rescue StandardError
+        []
+      end
+
+      def database_available?
+        SolidQueue::Record.connection_pool&.with_connection(&:active?)
+      rescue StandardError
+        false
+      end
+
+      def async_pool_recommendation
+        required_threads = configuration.estimated_number_of_threads
+        pool_size = SolidQueue::Record.connection_pool&.size
+        return [] if pool_size.nil? || required_threads <= pool_size
+
+        ["Increase queue DB pool to at least #{required_threads + 2} " \
+         "for async supervisor mode (currently #{pool_size})"]
+      rescue StandardError
+        []
+      end
+    end
+  end
+end

data/lib/solid_queue_guard/runner.rb

@@ -27,10 +27,19 @@ module SolidQueueGuard
     end
 
     def run_check(check_class)
+      check_id = Checks::Registry.check_id_for(check_class)
+      unless SolidQueueGuard.config.check_enabled?(check_id)
+        return Check::Result.new(
+          id: check_id,
+          status: :skip,
+          message: 'Check disabled via configuration'
+        )
+      end
+
       check_class.call
     rescue StandardError => e
       Check::Result.new(
-        id: check_class.name.demodulize.underscore,
+        id: check_id,
         status: :fail,
         message: "Check raised an error: #{e.class}: #{e.message}"
       )

data/lib/solid_queue_guard/tasks.rb

@@ -17,7 +17,7 @@ namespace :solid_queue_guard do
 
   desc 'Print machine-readable health status'
   task health: :environment do
-    SolidQueueGuard::CLI.run!(scope: :config, format: :json, strict: cli_strict)
+    SolidQueueGuard::CLI.run!(scope: :all, format: :json, strict: cli_strict)
   end
 
   desc 'Print full diagnostic report. Usage: report[json]'

data/lib/solid_queue_guard/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SolidQueueGuard
-  VERSION = '0.1.0'
+  VERSION = '1.0.0'
 end

data/lib/solid_queue_guard.rb

@@ -18,7 +18,7 @@ module SolidQueueGuard
     attr_accessor :configuration
 
     def deprecator
-      @deprecator ||= ActiveSupport::Deprecation.new('0.2', 'SolidQueueGuard')
+      @deprecator ||= ActiveSupport::Deprecation.new('2.0', 'SolidQueueGuard')
     end
 
     def configure

data/solid_queue_guard.gemspec

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative 'lib/solid_queue_guard/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = 'solid_queue_guard'
+  spec.version     = SolidQueueGuard::VERSION
+  spec.authors     = ['Rafael Pissardo']
+  spec.email       = ['rpissardo@users.noreply.github.com']
+  spec.homepage    = 'https://github.com/rafael-pissardo/solid_queue_guard'
+  spec.summary     = 'Production readiness checks and runtime guards for Rails Solid Queue.'
+  spec.description = [
+    'Detect queue lag, dead workers, unsafe thread/pool configuration,',
+    'and broken recurring jobs before they become incidents.'
+  ].join(' ')
+  spec.license = 'MIT'
+
+  spec.metadata = {
+    'homepage_uri' => spec.homepage,
+    'source_code_uri' => 'https://github.com/rafael-pissardo/solid_queue_guard',
+    'changelog_uri' => 'https://github.com/rafael-pissardo/solid_queue_guard/blob/main/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/rafael-pissardo/solid_queue_guard/issues',
+    'documentation_uri' => 'https://github.com/rafael-pissardo/solid_queue_guard#readme',
+    'rubygems_mfa_required' => 'true'
+  }
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |file|
+      file.start_with?('test/', '.github/', 'docs/', 'script/', 'gemfiles/', 'Appraisals')
+    end
+  end
+
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 3.1'
+
+  rails_version = '>= 7.1', '< 9.0'
+
+  spec.add_dependency 'actionpack', rails_version
+  spec.add_dependency 'activejob', rails_version
+  spec.add_dependency 'activerecord', rails_version
+  spec.add_dependency 'activesupport', rails_version
+  spec.add_dependency 'railties', rails_version
+  spec.add_dependency 'solid_queue', '>= 1.0', '< 2.0'
+
+  spec.add_development_dependency 'appraisal', '~> 2.5'
+  spec.add_development_dependency 'debug'
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'mocha', '>= 2.1'
+  spec.add_development_dependency 'puma', '>= 6.0'
+  spec.add_development_dependency 'rails', rails_version
+  spec.add_development_dependency 'rubocop', '~> 1.75'
+  spec.add_development_dependency 'rubocop-rails', '~> 2.30'
+  spec.add_development_dependency 'sqlite3', '~> 2.1'
+end