solid_queue_guard 0.1.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +14 -6
  3. data/app/controllers/solid_queue_guard/application_controller.rb +1 -0
  4. data/app/controllers/solid_queue_guard/health_controller.rb +25 -1
  5. data/lib/generators/solid_queue_guard/install/templates/solid_queue_guard.rb +5 -1
  6. data/lib/solid_queue_guard/checks/config/topology_recommendation_check.rb +24 -0
  7. data/lib/solid_queue_guard/checks/registry.rb +2 -1
  8. data/lib/solid_queue_guard/checks/runtime/base.rb +1 -5
  9. data/lib/solid_queue_guard/checks/runtime/blocked_jobs_check.rb +20 -1
  10. data/lib/solid_queue_guard/checks/runtime/database_support.rb +37 -0
  11. data/lib/solid_queue_guard/checks/runtime/dispatcher_check.rb +24 -1
  12. data/lib/solid_queue_guard/checks/runtime/failed_jobs_check.rb +21 -1
  13. data/lib/solid_queue_guard/checks/runtime/finished_jobs_growth_check.rb +20 -1
  14. data/lib/solid_queue_guard/checks/runtime/orphaned_claims_check.rb +18 -1
  15. data/lib/solid_queue_guard/checks/runtime/paused_queue_lag_check.rb +22 -1
  16. data/lib/solid_queue_guard/checks/runtime/pidfile_check.rb +33 -1
  17. data/lib/solid_queue_guard/checks/runtime/process_topology_check.rb +31 -1
  18. data/lib/solid_queue_guard/checks/runtime/queue_lag_check.rb +30 -1
  19. data/lib/solid_queue_guard/checks/runtime/recurring_stale_check.rb +29 -1
  20. data/lib/solid_queue_guard/checks/runtime/scheduled_backlog_check.rb +18 -1
  21. data/lib/solid_queue_guard/checks/runtime/stale_process_check.rb +19 -1
  22. data/lib/solid_queue_guard/cli.rb +10 -1
  23. data/lib/solid_queue_guard/configuration.rb +3 -1
  24. data/lib/solid_queue_guard/engine.rb +12 -0
  25. data/lib/solid_queue_guard/formatters/terminal.rb +1 -1
  26. data/lib/solid_queue_guard/health/cache.rb +28 -0
  27. data/lib/solid_queue_guard/health/payload.rb +50 -0
  28. data/lib/solid_queue_guard/metrics/exporter.rb +34 -0
  29. data/lib/solid_queue_guard/metrics/open_telemetry.rb +19 -0
  30. data/lib/solid_queue_guard/metrics/prometheus.rb +21 -0
  31. data/lib/solid_queue_guard/metrics/statsd.rb +25 -0
  32. data/lib/solid_queue_guard/notifier.rb +76 -0
  33. data/lib/solid_queue_guard/notifiers/base.rb +2 -7
  34. data/lib/solid_queue_guard/recommendations/topology.rb +63 -0
  35. data/lib/solid_queue_guard/tasks.rb +1 -1
  36. data/lib/solid_queue_guard/version.rb +1 -1
  37. metadata +11 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bd60b98c25c507b3069f98b6b69282f627a398e96bd43d7a5d0713f0499c8610
4
- data.tar.gz: c60a8995c311f4935456f58ca950e1d4cdb4b7e601dc1e6290f62f9d06931cb0
3
+ metadata.gz: c3b3fded32011114a722d7207eb15c44dad0d1be1e0c5b3202bdaf32f84ea819
4
+ data.tar.gz: 6ef2779a3372987138feb9a2dc9730ba7750c67a544a2c821029988877feb8de
5
5
  SHA512:
6
- metadata.gz: ac5fbbb57df0cb133466076f0c4c1b698a13004b3f44e22bc64139e04e8c4bb51973a22e534ff0617329709a154a92a7a93ac9e083ea0087d86dadc73c6e89cf
7
- data.tar.gz: b16bc3883213b26928b0cfa2c4d0b355ce43bd956a4c3df27e4cad4f8f71534748a15e5f9c892190a018741df8c9f1fbe4aaaa577e960f4e39a2a28c00a823ed
6
+ metadata.gz: 39c815be31c9ec06116840f813f57a6a9ebcb2dcad7c3747bf47d87cb59304fbdc6e629be2ffbb725eb649c913a75409c6b1e39bcb9c3e627ac00707af579e90
7
+ data.tar.gz: c338648964e042b7d5fb485c8281bc1337ab54acfe693b886e9f8ed656135aa924fe8cbc89af3382003dafea8844ab01497e4205b92ae474b376b1419751d68d
data/README.md CHANGED
@@ -138,7 +138,7 @@ Perfect for deploy pipelines:
138
138
  run: SOLID_QUEUE_GUARD_STRICT=1 bin/rails solid_queue_guard:doctor
139
139
  ```
140
140
 
141
- ### HTTP health (v0.2)
141
+ ### HTTP health
142
142
 
143
143
  ```ruby
144
144
  # config/routes.rb
@@ -150,6 +150,13 @@ curl localhost:3000/solid_queue_guard/health
150
150
  # => { "status": "degraded", "queue_lag_seconds": 245, "warnings": [...] }
151
151
  ```
152
152
 
153
+ Optional token protection:
154
+
155
+ ```ruby
156
+ config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
157
+ # curl -H "X-Solid-Queue-Guard-Token: $TOKEN" ...
158
+ ```
159
+
153
160
  Works with **Kamal**, **Heroku**, **Fly.io**, **ECS/Fargate**, **Kubernetes**, **Better Stack**, **UptimeRobot**.
154
161
 
155
162
  ---
@@ -197,11 +204,11 @@ end
197
204
 
198
205
  | Version | Ships |
199
206
  | ------- | ----- |
200
- | **v0.1** | `doctor` — config checks, CI integration, install generator |
201
- | **v0.2** | Runtime health, queue lag, dispatcher/blocked jobs, HTTP endpoint |
202
- | **v0.3** | Slack, Datadog, webhook notifications |
203
- | **v0.4** | StatsD, Prometheus, OpenTelemetry metrics |
204
- | **v0.5** | Auto-recommendations for `queue.yml` topology |
207
+ | **v0.1** | `doctor` — config checks, CI integration, install generator | ✅ Released |
208
+ | **v0.2** | Runtime health, queue lag, dispatcher/blocked jobs, HTTP endpoint | ✅ Released |
209
+ | **v0.3** | Slack, Datadog, webhook notifications | ✅ Released |
210
+ | **v0.4** | StatsD, Prometheus, OpenTelemetry metrics | ✅ Released |
211
+ | **v0.5** | Auto-recommendations for `queue.yml` topology | ✅ Released |
205
212
 
206
213
  ---
207
214
 
@@ -210,6 +217,7 @@ end
210
217
  | Gem version | Ruby | Rails |
211
218
  | ----------- | ---- | ----- |
212
219
  | 0.1.x | 3.1+ | 7.1, 7.2, 8.0 |
220
+ | 0.5.x | 3.1+ | 7.1, 7.2, 8.0 |
213
221
 
214
222
  ## Requirements
215
223
 
@@ -2,5 +2,6 @@
2
2
 
3
3
  module SolidQueueGuard
4
4
  class ApplicationController < ActionController::Base
5
+ protect_from_forgery with: :null_session
5
6
  end
6
7
  end
@@ -2,8 +2,32 @@
2
2
 
3
3
  module SolidQueueGuard
4
4
  class HealthController < ApplicationController
5
+ before_action :authenticate_token!
6
+
5
7
  def show
6
- render json: { status: 'not_implemented', message: 'HTTP health checks ship in v0.2' }, status: :not_implemented
8
+ payload = Health::Cache.fetch
9
+ status_code = http_status_for(payload[:status])
10
+
11
+ render json: payload, status: status_code
12
+ end
13
+
14
+ private
15
+
16
+ def authenticate_token!
17
+ token = SolidQueueGuard.config.health_token
18
+ return if token.blank?
19
+
20
+ provided = request.headers['X-Solid-Queue-Guard-Token'].presence || params[:token]
21
+ return if ActiveSupport::SecurityUtils.secure_compare(provided.to_s, token.to_s)
22
+
23
+ render json: { status: 'unauthorized' }, status: :unauthorized
24
+ end
25
+
26
+ def http_status_for(status)
27
+ case status.to_s
28
+ when 'unhealthy' then :service_unavailable
29
+ else :ok
30
+ end
7
31
  end
8
32
  end
9
33
  end
@@ -11,7 +11,11 @@ SolidQueueGuard.configure do |config|
11
11
 
12
12
  config.failed_jobs_threshold = 20
13
13
  config.stale_process_threshold = 5.minutes
14
+ config.health_cache_ttl = 15.seconds
15
+ config.scheduled_backlog_threshold = 100
14
16
 
15
17
  # config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
16
- # config.notify_with = [ :rails_logger ]
18
+ # config.integrate_rails_health = true
19
+ # config.notify_with = [:rails_logger, :slack, :datadog, :webhook]
20
+ # config.metrics_backends = [:statsd, :prometheus, :opentelemetry]
17
21
  end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Checks
5
+ module Config
6
+ class TopologyRecommendationCheck < Base
7
+ def call
8
+ recommendations = Recommendations::Topology.analyze
9
+
10
+ if recommendations.empty?
11
+ pass('topology_recommendation', 'No queue.yml topology changes recommended')
12
+ else
13
+ warn(
14
+ 'topology_recommendation',
15
+ recommendations.join('; '),
16
+ suggestion: 'Review config/queue.yml worker and database pool settings',
17
+ metadata: { recommendations: recommendations }
18
+ )
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -14,7 +14,8 @@ module SolidQueueGuard
14
14
  Config::SchedulerConfigCheck,
15
15
  Config::EnvFlagsCheck,
16
16
  Config::ProcessHeartbeatConfigCheck,
17
- Config::PumaColocatedCheck
17
+ Config::PumaColocatedCheck,
18
+ Config::TopologyRecommendationCheck
18
19
  ].freeze
19
20
 
20
21
  RUNTIME_CHECKS = [
@@ -4,11 +4,7 @@ module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
6
  class Base < Checks::Base
7
- NOT_IMPLEMENTED_MESSAGE = 'Runtime check not implemented until v0.2'
8
-
9
- def call
10
- skip(check_id, NOT_IMPLEMENTED_MESSAGE)
11
- end
7
+ include DatabaseSupport
12
8
 
13
9
  private
14
10
 
@@ -3,7 +3,26 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class BlockedJobsCheck < Base; end
6
+ class BlockedJobsCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ blocked = SolidQueue::BlockedExecution.count
10
+ expired = SolidQueue::BlockedExecution.expired.count
11
+
12
+ if expired.positive?
13
+ warn(
14
+ check_id,
15
+ "#{expired} blocked job(s) have expired concurrency locks",
16
+ suggestion: 'Verify concurrency release maintenance is running'
17
+ )
18
+ elsif blocked.positive?
19
+ pass(check_id, "#{blocked} job(s) blocked by concurrency control")
20
+ else
21
+ pass(check_id, 'No blocked jobs')
22
+ end
23
+ end
24
+ end
25
+ end
7
26
  end
8
27
  end
9
28
  end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Checks
5
+ module Runtime
6
+ module DatabaseSupport
7
+ private
8
+
9
+ def queue_database_available?
10
+ pool = SolidQueue::Record.connection_pool
11
+ return false if pool.nil?
12
+
13
+ pool.with_connection(&:active?)
14
+ rescue StandardError
15
+ false
16
+ end
17
+
18
+ def with_queue_database
19
+ return skip(check_id, 'Queue database is not available') unless queue_database_available?
20
+
21
+ yield
22
+ rescue StandardError => e
23
+ skip(check_id, "Queue database query failed: #{e.class}")
24
+ end
25
+
26
+ def config
27
+ SolidQueueGuard.config
28
+ end
29
+
30
+ def lag_threshold_for(queue_name)
31
+ thresholds = config.queue_lag_thresholds
32
+ thresholds[queue_name.to_sym] || thresholds[queue_name] || thresholds[:default]
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -3,7 +3,30 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class DispatcherCheck < Base; end
6
+ class DispatcherCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ dispatchers = SolidQueue::Process.where(kind: 'Dispatcher')
10
+ due_count = SolidQueue::ScheduledExecution.due.count
11
+
12
+ if dispatchers.none? && due_count.positive?
13
+ failure(
14
+ check_id,
15
+ "#{due_count} scheduled job(s) are due but no dispatcher is running",
16
+ suggestion: 'Start a dispatcher process or verify bin/jobs is running'
17
+ )
18
+ elsif due_count > config.scheduled_backlog_threshold
19
+ warn(
20
+ check_id,
21
+ "#{due_count} scheduled executions are due (threshold: #{config.scheduled_backlog_threshold})",
22
+ suggestion: 'Verify the dispatcher is keeping up with scheduled work'
23
+ )
24
+ else
25
+ pass(check_id, "Dispatcher healthy (#{due_count} due scheduled executions)")
26
+ end
27
+ end
28
+ end
29
+ end
7
30
  end
8
31
  end
9
32
  end
@@ -3,7 +3,27 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class FailedJobsCheck < Base; end
6
+ class FailedJobsCheck < Base
7
+ WINDOW = 1.hour
8
+
9
+ def call
10
+ with_queue_database do
11
+ count = SolidQueue::FailedExecution.where(created_at: WINDOW.ago..).count
12
+ threshold = config.failed_jobs_threshold
13
+
14
+ if count > threshold
15
+ warn(
16
+ check_id,
17
+ "#{count} failed job(s) in the last hour (threshold: #{threshold})",
18
+ suggestion: 'Review failed jobs in Mission Control or solid_queue_failed_executions',
19
+ metadata: { failed_jobs_last_hour: count }
20
+ )
21
+ else
22
+ pass(check_id, "#{count} failed job(s) in the last hour")
23
+ end
24
+ end
25
+ end
26
+ end
7
27
  end
8
28
  end
9
29
  end
@@ -3,7 +3,26 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class FinishedJobsGrowthCheck < Base; end
6
+ class FinishedJobsGrowthCheck < Base
7
+ WINDOW = 24.hours
8
+ GROWTH_THRESHOLD = 10_000
9
+
10
+ def call
11
+ with_queue_database do
12
+ recent_finished = SolidQueue::Job.where(finished_at: WINDOW.ago..).count
13
+
14
+ if recent_finished > GROWTH_THRESHOLD
15
+ warn(
16
+ check_id,
17
+ "#{recent_finished} finished jobs in the last 24 hours",
18
+ suggestion: 'Ensure a recurring cleanup task runs (SolidQueue::Job.clear_finished_in_batches)'
19
+ )
20
+ else
21
+ pass(check_id, "#{recent_finished} finished jobs in the last 24 hours")
22
+ end
23
+ end
24
+ end
25
+ end
7
26
  end
8
27
  end
9
28
  end
@@ -3,7 +3,24 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class OrphanedClaimsCheck < Base; end
6
+ class OrphanedClaimsCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ alive_process_ids = SolidQueue::Process.pluck(:id)
10
+ orphaned = SolidQueue::ClaimedExecution.where.not(process_id: alive_process_ids)
11
+
12
+ if orphaned.none?
13
+ pass(check_id, 'No orphaned claimed executions')
14
+ else
15
+ warn(
16
+ check_id,
17
+ "#{orphaned.count} claimed execution(s) belong to dead processes",
18
+ suggestion: 'Restart workers; Solid Queue maintenance should release orphaned claims'
19
+ )
20
+ end
21
+ end
22
+ end
23
+ end
7
24
  end
8
25
  end
9
26
  end
@@ -3,7 +3,28 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class PausedQueueLagCheck < Base; end
6
+ class PausedQueueLagCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ paused_queues = SolidQueue::Pause.pluck(:queue_name)
10
+ return pass(check_id, 'No paused queues') if paused_queues.empty?
11
+
12
+ growing = paused_queues.select do |queue_name|
13
+ SolidQueue::ReadyExecution.exists?(queue_name: queue_name)
14
+ end
15
+
16
+ if growing.any?
17
+ warn(
18
+ check_id,
19
+ "Paused queue(s) still have ready jobs: #{growing.join(', ')}",
20
+ suggestion: 'Unpause queues or drain jobs before pausing'
21
+ )
22
+ else
23
+ pass(check_id, "Paused queues have no ready backlog: #{paused_queues.join(', ')}")
24
+ end
25
+ end
26
+ end
27
+ end
7
28
  end
8
29
  end
9
30
  end
@@ -3,7 +3,39 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class PidfileCheck < Base; end
6
+ class PidfileCheck < Base
7
+ def call
8
+ pidfile = ENV['SOLID_QUEUE_PIDFILE'] || default_pidfile
9
+ return pass(check_id, 'No Solid Queue pidfile configured') if pidfile.blank?
10
+
11
+ path = Pathname(pidfile)
12
+ unless path.exist?
13
+ return warn(check_id, "Pidfile not found at #{path}",
14
+ suggestion: 'Verify Solid Queue supervisor is running')
15
+ end
16
+
17
+ pid = path.read.strip.to_i
18
+ if process_alive?(pid)
19
+ pass(check_id, "Solid Queue supervisor pidfile present (pid #{pid})")
20
+ else
21
+ failure(check_id, "Pidfile exists but process #{pid} is not running",
22
+ suggestion: 'Restart the Solid Queue supervisor')
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ def default_pidfile
29
+ rails_root.join('tmp/pids/solid_queue.pid').to_s
30
+ end
31
+
32
+ def process_alive?(pid)
33
+ Process.getpgid(pid)
34
+ true
35
+ rescue Errno::ESRCH
36
+ false
37
+ end
38
+ end
7
39
  end
8
40
  end
9
41
  end
@@ -3,7 +3,37 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class ProcessTopologyCheck < Base; end
6
+ class ProcessTopologyCheck < Base
7
+ EXPECTED_KINDS = %w[Supervisor Worker Dispatcher Scheduler].freeze
8
+
9
+ def call
10
+ with_queue_database do
11
+ kinds = SolidQueue::Process.distinct.pluck(:kind)
12
+ missing = EXPECTED_KINDS - kinds
13
+
14
+ if missing == EXPECTED_KINDS
15
+ warn(check_id, 'No Solid Queue processes are running',
16
+ suggestion: 'Start bin/jobs or the Solid Queue supervisor')
17
+ elsif missing.include?('Worker')
18
+ warn(check_id, 'No worker processes detected',
19
+ suggestion: 'Ensure workers are configured in queue.yml and running')
20
+ elsif missing.include?('Dispatcher') && scheduled_work_expected?
21
+ warn(check_id, 'No dispatcher process detected with scheduled work configured')
22
+ else
23
+ pass(check_id, "Solid Queue processes running: #{kinds.sort.join(', ')}")
24
+ end
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def scheduled_work_expected?
31
+ solid_queue_configuration.send(:recurring_tasks).any? ||
32
+ SolidQueue::ScheduledExecution.exists?
33
+ rescue StandardError
34
+ false
35
+ end
36
+ end
7
37
  end
8
38
  end
9
39
  end
@@ -3,7 +3,36 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class QueueLagCheck < Base; end
6
+ class QueueLagCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ lags = SolidQueue::ReadyExecution.group(:queue_name).minimum(:created_at)
10
+ return pass(check_id, 'No jobs waiting in ready queues') if lags.empty?
11
+
12
+ worst = lags.map do |queue_name, oldest|
13
+ lag = Time.current - oldest
14
+ { queue: queue_name, lag: lag, threshold: lag_threshold_for(queue_name) }
15
+ end.max_by { |entry| entry[:lag] }
16
+
17
+ if worst[:lag] > worst[:threshold]
18
+ failure(
19
+ check_id,
20
+ "#{worst[:queue]} queue lag is #{worst[:lag].to_i} seconds",
21
+ suggestion: "Investigate worker coverage and throughput for the #{worst[:queue]} queue",
22
+ metadata: { queue_lag_seconds: worst[:lag].to_i, queue_name: worst[:queue] }
23
+ )
24
+ elsif worst[:lag] > worst[:threshold] / 2
25
+ warn(
26
+ check_id,
27
+ "#{worst[:queue]} queue lag is #{worst[:lag].to_i} seconds",
28
+ metadata: { queue_lag_seconds: worst[:lag].to_i, queue_name: worst[:queue] }
29
+ )
30
+ else
31
+ pass(check_id, "Queue lag within thresholds (worst: #{worst[:queue]} at #{worst[:lag].to_i}s)")
32
+ end
33
+ end
34
+ end
35
+ end
7
36
  end
8
37
  end
9
38
  end
@@ -3,7 +3,35 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class RecurringStaleCheck < Base; end
6
+ class RecurringStaleCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ tasks = SolidQueue::RecurringTask.static
10
+ return pass(check_id, 'No recurring tasks configured') if tasks.none?
11
+
12
+ stale_tasks = tasks.select do |task|
13
+ last_run = SolidQueue::RecurringExecution.where(task_key: task.key).maximum(:run_at)
14
+ last_run.nil? || last_run < expected_staleness.ago
15
+ end
16
+
17
+ if stale_tasks.any?
18
+ warn(
19
+ check_id,
20
+ "#{stale_tasks.size} recurring task(s) may be stale: #{stale_tasks.map(&:key).join(', ')}",
21
+ suggestion: 'Verify the scheduler process is running'
22
+ )
23
+ else
24
+ pass(check_id, 'Recurring tasks have recent executions')
25
+ end
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ def expected_staleness
32
+ config.stale_process_threshold * 2
33
+ end
34
+ end
7
35
  end
8
36
  end
9
37
  end
@@ -3,7 +3,24 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class ScheduledBacklogCheck < Base; end
6
+ class ScheduledBacklogCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ due_count = SolidQueue::ScheduledExecution.due.count
10
+ threshold = config.scheduled_backlog_threshold
11
+
12
+ if due_count > threshold
13
+ warn(
14
+ check_id,
15
+ "Scheduled backlog is #{due_count} (threshold: #{threshold})",
16
+ suggestion: 'Check dispatcher health and scheduled job volume'
17
+ )
18
+ else
19
+ pass(check_id, "Scheduled backlog is #{due_count}")
20
+ end
21
+ end
22
+ end
23
+ end
7
24
  end
8
25
  end
9
26
  end
@@ -3,7 +3,25 @@
3
3
  module SolidQueueGuard
4
4
  module Checks
5
5
  module Runtime
6
- class StaleProcessCheck < Base; end
6
+ class StaleProcessCheck < Base
7
+ def call
8
+ with_queue_database do
9
+ threshold = config.stale_process_threshold
10
+ stale = SolidQueue::Process.where(last_heartbeat_at: ...threshold.ago)
11
+
12
+ if stale.none?
13
+ pass(check_id, 'All Solid Queue processes have recent heartbeats')
14
+ else
15
+ failure(
16
+ check_id,
17
+ "#{stale.count} Solid Queue process(es) have stale heartbeats",
18
+ suggestion: 'Restart the Solid Queue supervisor and verify workers are running',
19
+ metadata: { stale_processes: stale.count }
20
+ )
21
+ end
22
+ end
23
+ end
24
+ end
7
25
  end
8
26
  end
9
27
  end
@@ -18,9 +18,12 @@ module SolidQueueGuard
18
18
  strict = strict_flag?(argv) if strict.nil?
19
19
 
20
20
  report = Runner.new(scope: scope).run
21
+ Notifier.deliver_all(report) if notify?(report)
22
+ Metrics::Exporter.export(report) if SolidQueueGuard.config.metrics_backends.any?
23
+
21
24
  output = formatter_for(format).new(report).render
22
25
 
23
- Rails.logger.debug output unless output.empty?
26
+ $stdout.puts(output) unless output.empty?
24
27
 
25
28
  exit report.exit_code(strict: strict)
26
29
  end
@@ -51,5 +54,11 @@ module SolidQueueGuard
51
54
  ENV.fetch('SOLID_QUEUE_GUARD_SCOPE', 'config').to_sym
52
55
  end
53
56
  end
57
+
58
+ def notify?(report)
59
+ return false unless SolidQueueGuard.config.notify_with.any?
60
+
61
+ report.status != :healthy
62
+ end
54
63
  end
55
64
  end
@@ -11,7 +11,8 @@ module SolidQueueGuard
11
11
  :health_cache_ttl,
12
12
  :scheduled_backlog_threshold,
13
13
  :integrate_rails_health,
14
- :notify_with
14
+ :notify_with,
15
+ :metrics_backends
15
16
 
16
17
  def initialize
17
18
  @enabled = true
@@ -24,6 +25,7 @@ module SolidQueueGuard
24
25
  @scheduled_backlog_threshold = 100
25
26
  @integrate_rails_health = false
26
27
  @notify_with = [:rails_logger]
28
+ @metrics_backends = []
27
29
  end
28
30
 
29
31
  def strict?
@@ -17,5 +17,17 @@ module SolidQueueGuard
17
17
  end
18
18
  end
19
19
  end
20
+
21
+ initializer 'solid_queue_guard.rails_health' do
22
+ next unless SolidQueueGuard.config.integrate_rails_health
23
+
24
+ ActiveSupport.on_load(:action_controller_base) do
25
+ Rails::HealthController.class_eval do
26
+ def solid_queue_guard_status
27
+ SolidQueueGuard::Health::Cache.fetch[:status]
28
+ end
29
+ end
30
+ end
31
+ end
20
32
  end
21
33
  end
@@ -27,7 +27,7 @@ module SolidQueueGuard
27
27
  attr_reader :report
28
28
 
29
29
  def check_lines
30
- report.results.map do |result|
30
+ report.results.reject(&:skip?).map do |result|
31
31
  icon = ICONS.fetch(result.status, '•')
32
32
  "#{icon} #{result.message}"
33
33
  end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Health
5
+ class Cache
6
+ def self.fetch
7
+ new.fetch
8
+ end
9
+
10
+ def fetch
11
+ Rails.cache.fetch(cache_key, expires_in: SolidQueueGuard.config.health_cache_ttl) do
12
+ build_payload
13
+ end
14
+ end
15
+
16
+ private
17
+
18
+ def cache_key
19
+ 'solid_queue_guard/health'
20
+ end
21
+
22
+ def build_payload
23
+ report = Runner.new(scope: :all).run
24
+ Payload.new(report).to_h
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Health
5
+ class Payload
6
+ def initialize(report)
7
+ @report = report
8
+ end
9
+
10
+ def to_h
11
+ base = report.to_h.merge(
12
+ queue_lag_seconds: queue_lag_seconds,
13
+ failed_jobs_last_hour: failed_jobs_last_hour,
14
+ dead_processes: dead_processes
15
+ )
16
+
17
+ base[:recommendations] = topology_recommendations if topology_recommendations.any?
18
+ base
19
+ end
20
+
21
+ private
22
+
23
+ attr_reader :report
24
+
25
+ def queue_lag_seconds
26
+ metadata_value('queue_lag') { |metadata| metadata[:queue_lag_seconds] }
27
+ end
28
+
29
+ def failed_jobs_last_hour
30
+ metadata_value('failed_jobs') { |metadata| metadata[:failed_jobs_last_hour] } || 0
31
+ end
32
+
33
+ def dead_processes
34
+ metadata_value('stale_process') { |metadata| metadata[:stale_processes] } || 0
35
+ end
36
+
37
+ def metadata_value(check_id)
38
+ result = report.results.find { |entry| entry.id == check_id }
39
+ metadata = result&.metadata || {}
40
+ value = yield(metadata)
41
+ value unless value.nil?
42
+ end
43
+
44
+ def topology_recommendations
45
+ result = report.results.find { |entry| entry.id == 'topology_recommendation' }
46
+ result&.metadata&.fetch(:recommendations, []) || []
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Metrics
5
+ class Exporter
6
+ STATUS_VALUES = { healthy: 0, degraded: 1, unhealthy: 2 }.freeze
7
+
8
+ def self.export(report, backends: SolidQueueGuard.config.metrics_backends)
9
+ new(report, backends: backends).export
10
+ end
11
+
12
+ def initialize(report, backends:)
13
+ @report = report
14
+ @backends = Array(backends)
15
+ end
16
+
17
+ def export
18
+ backends.each { |backend| export_backend(backend) }
19
+ end
20
+
21
+ private
22
+
23
+ attr_reader :report, :backends
24
+
25
+ def export_backend(backend)
26
+ case backend.to_sym
27
+ when :statsd then Statsd.export(report)
28
+ when :prometheus then Prometheus.export(report)
29
+ when :opentelemetry then OpenTelemetry.export(report)
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Metrics
5
+ module OpenTelemetry
6
+ module_function
7
+
8
+ def export(report)
9
+ return unless defined?(::OpenTelemetry)
10
+
11
+ meter = ::OpenTelemetry.meter_provider.meter('solid_queue_guard')
12
+ gauge = meter.create_gauge('solid_queue.guard.overall_status', unit: 'status')
13
+ gauge.record(Exporter::STATUS_VALUES.fetch(report.status))
14
+ rescue StandardError
15
+ nil
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Metrics
5
+ module Prometheus
6
+ module_function
7
+
8
+ def export(report)
9
+ path = ENV.fetch('SOLID_QUEUE_GUARD_PROMETHEUS_FILE', Rails.root.join('tmp/solid_queue_guard.prom'))
10
+ File.write(path, render(report))
11
+ end
12
+
13
+ def render(report)
14
+ <<~PROM
15
+ # TYPE solid_queue_guard_overall_status gauge
16
+ solid_queue_guard_overall_status #{Exporter::STATUS_VALUES.fetch(report.status)}
17
+ PROM
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Metrics
5
+ module Statsd
6
+ module_function
7
+
8
+ def export(report)
9
+ require 'socket'
10
+ host = ENV.fetch('SOLID_QUEUE_GUARD_STATSD_HOST', '127.0.0.1')
11
+ port = ENV.fetch('SOLID_QUEUE_GUARD_STATSD_PORT', 8125).to_i
12
+ socket = UDPSocket.new
13
+ socket.send(metric_line(report), 0, host, port)
14
+ ensure
15
+ socket&.close
16
+ end
17
+
18
+ def metric_line(report)
19
+ "solid_queue.guard.overall_status:#{STATUS_VALUES.fetch(report.status)}|g"
20
+ end
21
+
22
+ STATUS_VALUES = Exporter::STATUS_VALUES
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ class Notifier
5
+ class << self
6
+ def deliver_all(report, adapters: SolidQueueGuard.config.notify_with)
7
+ Array(adapters).each do |adapter|
8
+ deliver(adapter, report)
9
+ end
10
+ end
11
+
12
+ def deliver(adapter, report)
13
+ case adapter.to_sym
14
+ when :slack then deliver_slack(report)
15
+ when :datadog then deliver_datadog(report)
16
+ when :webhook then deliver_webhook(report)
17
+ else deliver_rails_logger(report)
18
+ end
19
+ end
20
+
21
+ def post_json(url, payload, headers: {})
22
+ require 'net/http'
23
+ uri = URI(url)
24
+ request = Net::HTTP::Post.new(uri)
25
+ request['Content-Type'] = 'application/json'
26
+ headers.each { |key, value| request[key] = value }
27
+ request.body = JSON.generate(payload)
28
+ Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
29
+ http.request(request)
30
+ end
31
+ end
32
+
33
+ private
34
+
35
+ def deliver_rails_logger(report)
36
+ Rails.logger.warn("[SolidQueueGuard] status=#{report.status} warnings=#{report.warnings.join('; ')}")
37
+ end
38
+
39
+ def deliver_slack(report)
40
+ webhook_url = ENV.fetch('SOLID_QUEUE_GUARD_SLACK_WEBHOOK_URL', nil)
41
+ return if webhook_url.blank?
42
+
43
+ post_json(webhook_url, { text: "SolidQueueGuard status: *#{report.status}*\n#{report.warnings.join("\n")}" })
44
+ end
45
+
46
+ def deliver_datadog(report)
47
+ api_key = ENV.fetch('DD_API_KEY', nil)
48
+ return if api_key.blank?
49
+
50
+ alert_type = case report.status
51
+ when :unhealthy then 'error'
52
+ when :degraded then 'warning'
53
+ else 'info'
54
+ end
55
+
56
+ post_json(
57
+ 'https://api.datadoghq.com/api/v1/events',
58
+ {
59
+ title: 'SolidQueueGuard alert',
60
+ text: report.warnings.join("\n"),
61
+ alert_type: alert_type,
62
+ tags: ["status:#{report.status}"]
63
+ },
64
+ headers: { 'DD-API-KEY' => api_key }
65
+ )
66
+ end
67
+
68
+ def deliver_webhook(report)
69
+ webhook_url = ENV.fetch('SOLID_QUEUE_GUARD_WEBHOOK_URL', nil)
70
+ return if webhook_url.blank?
71
+
72
+ post_json(webhook_url, report.to_h)
73
+ end
74
+ end
75
+ end
76
+ end
@@ -4,14 +4,9 @@ module SolidQueueGuard
4
4
  module Notifiers
5
5
  # @api private
6
6
  class Base
7
- def self.deliver(_report)
8
- raise NotImplementedError, 'Notification adapters ship in v0.3'
7
+ def self.deliver(report)
8
+ Notifier.deliver(:rails_logger, report)
9
9
  end
10
10
  end
11
-
12
- class RailsLogger < Base; end
13
- class Slack < Base; end
14
- class Datadog < Base; end
15
- class Webhook < Base; end
16
11
  end
17
12
  end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SolidQueueGuard
4
+ module Recommendations
5
+ class Topology
6
+ def self.analyze(configuration: SolidQueue::Configuration.new)
7
+ new(configuration: configuration).analyze
8
+ end
9
+
10
+ def initialize(configuration:)
11
+ @configuration = configuration
12
+ end
13
+
14
+ def analyze
15
+ recommendations = []
16
+ recommendations.concat(worker_recommendations)
17
+ recommendations.concat(pool_recommendations)
18
+ recommendations.uniq
19
+ end
20
+
21
+ private
22
+
23
+ attr_reader :configuration
24
+
25
+ def worker_recommendations
26
+ worker_queues = QueueCoverage.worker_queues_from_configuration(configuration)
27
+ uncovered = uncovered_queues(worker_queues)
28
+ return [] if uncovered.empty?
29
+
30
+ ["Add worker coverage for: #{uncovered.join(', ')}"]
31
+ end
32
+
33
+ def pool_recommendations
34
+ required_threads = configuration.estimated_number_of_threads
35
+ pool_size = SolidQueue::Record.connection_pool&.size
36
+ return [] if pool_size.nil? || required_threads <= pool_size
37
+
38
+ ["Increase queue DB pool to at least #{required_threads + 2} (currently #{pool_size})"]
39
+ rescue StandardError
40
+ []
41
+ end
42
+
43
+ def uncovered_queues(assigned_queues)
44
+ observed = observed_queues
45
+ (observed - assigned_queues).sort
46
+ end
47
+
48
+ def observed_queues
49
+ return [] unless database_available?
50
+
51
+ SolidQueue::Job.distinct.pluck(:queue_name)
52
+ rescue StandardError
53
+ []
54
+ end
55
+
56
+ def database_available?
57
+ SolidQueue::Record.connection_pool&.with_connection(&:active?)
58
+ rescue StandardError
59
+ false
60
+ end
61
+ end
62
+ end
63
+ end
@@ -17,7 +17,7 @@ namespace :solid_queue_guard do
17
17
 
18
18
  desc 'Print machine-readable health status'
19
19
  task health: :environment do
20
- SolidQueueGuard::CLI.run!(scope: :config, format: :json, strict: cli_strict)
20
+ SolidQueueGuard::CLI.run!(scope: :all, format: :json, strict: cli_strict)
21
21
  end
22
22
 
23
23
  desc 'Print full diagnostic report. Usage: report[json]'
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SolidQueueGuard
4
- VERSION = '0.1.0'
4
+ VERSION = '0.5.0'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: solid_queue_guard
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rafael Pissardo
@@ -290,10 +290,12 @@ files:
290
290
  - lib/solid_queue_guard/checks/config/queue_schema_check.rb
291
291
  - lib/solid_queue_guard/checks/config/scheduler_config_check.rb
292
292
  - lib/solid_queue_guard/checks/config/thread_pool_check.rb
293
+ - lib/solid_queue_guard/checks/config/topology_recommendation_check.rb
293
294
  - lib/solid_queue_guard/checks/config/worker_coverage_check.rb
294
295
  - lib/solid_queue_guard/checks/registry.rb
295
296
  - lib/solid_queue_guard/checks/runtime/base.rb
296
297
  - lib/solid_queue_guard/checks/runtime/blocked_jobs_check.rb
298
+ - lib/solid_queue_guard/checks/runtime/database_support.rb
297
299
  - lib/solid_queue_guard/checks/runtime/dispatcher_check.rb
298
300
  - lib/solid_queue_guard/checks/runtime/failed_jobs_check.rb
299
301
  - lib/solid_queue_guard/checks/runtime/finished_jobs_growth_check.rb
@@ -310,8 +312,16 @@ files:
310
312
  - lib/solid_queue_guard/engine.rb
311
313
  - lib/solid_queue_guard/formatters/json.rb
312
314
  - lib/solid_queue_guard/formatters/terminal.rb
315
+ - lib/solid_queue_guard/health/cache.rb
316
+ - lib/solid_queue_guard/health/payload.rb
317
+ - lib/solid_queue_guard/metrics/exporter.rb
318
+ - lib/solid_queue_guard/metrics/open_telemetry.rb
319
+ - lib/solid_queue_guard/metrics/prometheus.rb
320
+ - lib/solid_queue_guard/metrics/statsd.rb
321
+ - lib/solid_queue_guard/notifier.rb
313
322
  - lib/solid_queue_guard/notifiers/base.rb
314
323
  - lib/solid_queue_guard/queue_coverage.rb
324
+ - lib/solid_queue_guard/recommendations/topology.rb
315
325
  - lib/solid_queue_guard/report.rb
316
326
  - lib/solid_queue_guard/runner.rb
317
327
  - lib/solid_queue_guard/tasks.rb