solid_queue_guard 0.1.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +14 -6
- data/app/controllers/solid_queue_guard/application_controller.rb +1 -0
- data/app/controllers/solid_queue_guard/health_controller.rb +25 -1
- data/lib/generators/solid_queue_guard/install/templates/solid_queue_guard.rb +5 -1
- data/lib/solid_queue_guard/checks/config/topology_recommendation_check.rb +24 -0
- data/lib/solid_queue_guard/checks/registry.rb +2 -1
- data/lib/solid_queue_guard/checks/runtime/base.rb +1 -5
- data/lib/solid_queue_guard/checks/runtime/blocked_jobs_check.rb +20 -1
- data/lib/solid_queue_guard/checks/runtime/database_support.rb +37 -0
- data/lib/solid_queue_guard/checks/runtime/dispatcher_check.rb +24 -1
- data/lib/solid_queue_guard/checks/runtime/failed_jobs_check.rb +21 -1
- data/lib/solid_queue_guard/checks/runtime/finished_jobs_growth_check.rb +20 -1
- data/lib/solid_queue_guard/checks/runtime/orphaned_claims_check.rb +18 -1
- data/lib/solid_queue_guard/checks/runtime/paused_queue_lag_check.rb +22 -1
- data/lib/solid_queue_guard/checks/runtime/pidfile_check.rb +33 -1
- data/lib/solid_queue_guard/checks/runtime/process_topology_check.rb +31 -1
- data/lib/solid_queue_guard/checks/runtime/queue_lag_check.rb +30 -1
- data/lib/solid_queue_guard/checks/runtime/recurring_stale_check.rb +29 -1
- data/lib/solid_queue_guard/checks/runtime/scheduled_backlog_check.rb +18 -1
- data/lib/solid_queue_guard/checks/runtime/stale_process_check.rb +19 -1
- data/lib/solid_queue_guard/cli.rb +10 -1
- data/lib/solid_queue_guard/configuration.rb +3 -1
- data/lib/solid_queue_guard/engine.rb +12 -0
- data/lib/solid_queue_guard/formatters/terminal.rb +1 -1
- data/lib/solid_queue_guard/health/cache.rb +28 -0
- data/lib/solid_queue_guard/health/payload.rb +50 -0
- data/lib/solid_queue_guard/metrics/exporter.rb +34 -0
- data/lib/solid_queue_guard/metrics/open_telemetry.rb +19 -0
- data/lib/solid_queue_guard/metrics/prometheus.rb +21 -0
- data/lib/solid_queue_guard/metrics/statsd.rb +25 -0
- data/lib/solid_queue_guard/notifier.rb +76 -0
- data/lib/solid_queue_guard/notifiers/base.rb +2 -7
- data/lib/solid_queue_guard/recommendations/topology.rb +63 -0
- data/lib/solid_queue_guard/tasks.rb +1 -1
- data/lib/solid_queue_guard/version.rb +1 -1
- metadata +11 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c3b3fded32011114a722d7207eb15c44dad0d1be1e0c5b3202bdaf32f84ea819
|
|
4
|
+
data.tar.gz: 6ef2779a3372987138feb9a2dc9730ba7750c67a544a2c821029988877feb8de
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 39c815be31c9ec06116840f813f57a6a9ebcb2dcad7c3747bf47d87cb59304fbdc6e629be2ffbb725eb649c913a75409c6b1e39bcb9c3e627ac00707af579e90
|
|
7
|
+
data.tar.gz: c338648964e042b7d5fb485c8281bc1337ab54acfe693b886e9f8ed656135aa924fe8cbc89af3382003dafea8844ab01497e4205b92ae474b376b1419751d68d
|
data/README.md
CHANGED
|
@@ -138,7 +138,7 @@ Perfect for deploy pipelines:
|
|
|
138
138
|
run: SOLID_QUEUE_GUARD_STRICT=1 bin/rails solid_queue_guard:doctor
|
|
139
139
|
```
|
|
140
140
|
|
|
141
|
-
### HTTP health
|
|
141
|
+
### HTTP health
|
|
142
142
|
|
|
143
143
|
```ruby
|
|
144
144
|
# config/routes.rb
|
|
@@ -150,6 +150,13 @@ curl localhost:3000/solid_queue_guard/health
|
|
|
150
150
|
# => { "status": "degraded", "queue_lag_seconds": 245, "warnings": [...] }
|
|
151
151
|
```
|
|
152
152
|
|
|
153
|
+
Optional token protection:
|
|
154
|
+
|
|
155
|
+
```ruby
|
|
156
|
+
config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
|
|
157
|
+
# curl -H "X-Solid-Queue-Guard-Token: $TOKEN" ...
|
|
158
|
+
```
|
|
159
|
+
|
|
153
160
|
Works with **Kamal**, **Heroku**, **Fly.io**, **ECS/Fargate**, **Kubernetes**, **Better Stack**, **UptimeRobot**.
|
|
154
161
|
|
|
155
162
|
---
|
|
@@ -197,11 +204,11 @@ end
|
|
|
197
204
|
|
|
198
205
|
| Version | Ships |
|
|
199
206
|
| ------- | ----- |
|
|
200
|
-
| **v0.1** | `doctor` — config checks, CI integration, install generator |
|
|
201
|
-
| **v0.2** | Runtime health, queue lag, dispatcher/blocked jobs, HTTP endpoint |
|
|
202
|
-
| **v0.3** | Slack, Datadog, webhook notifications |
|
|
203
|
-
| **v0.4** | StatsD, Prometheus, OpenTelemetry metrics |
|
|
204
|
-
| **v0.5** | Auto-recommendations for `queue.yml` topology |
|
|
207
|
+
| **v0.1** | `doctor` — config checks, CI integration, install generator | ✅ Released |
|
|
208
|
+
| **v0.2** | Runtime health, queue lag, dispatcher/blocked jobs, HTTP endpoint | ✅ Released |
|
|
209
|
+
| **v0.3** | Slack, Datadog, webhook notifications | ✅ Released |
|
|
210
|
+
| **v0.4** | StatsD, Prometheus, OpenTelemetry metrics | ✅ Released |
|
|
211
|
+
| **v0.5** | Auto-recommendations for `queue.yml` topology | ✅ Released |
|
|
205
212
|
|
|
206
213
|
---
|
|
207
214
|
|
|
@@ -210,6 +217,7 @@ end
|
|
|
210
217
|
| Gem version | Ruby | Rails |
|
|
211
218
|
| ----------- | ---- | ----- |
|
|
212
219
|
| 0.1.x | 3.1+ | 7.1, 7.2, 8.0 |
|
|
220
|
+
| 0.5.x | 3.1+ | 7.1, 7.2, 8.0 |
|
|
213
221
|
|
|
214
222
|
## Requirements
|
|
215
223
|
|
|
@@ -2,8 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
class HealthController < ApplicationController
|
|
5
|
+
before_action :authenticate_token!
|
|
6
|
+
|
|
5
7
|
def show
|
|
6
|
-
|
|
8
|
+
payload = Health::Cache.fetch
|
|
9
|
+
status_code = http_status_for(payload[:status])
|
|
10
|
+
|
|
11
|
+
render json: payload, status: status_code
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
def authenticate_token!
|
|
17
|
+
token = SolidQueueGuard.config.health_token
|
|
18
|
+
return if token.blank?
|
|
19
|
+
|
|
20
|
+
provided = request.headers['X-Solid-Queue-Guard-Token'].presence || params[:token]
|
|
21
|
+
return if ActiveSupport::SecurityUtils.secure_compare(provided.to_s, token.to_s)
|
|
22
|
+
|
|
23
|
+
render json: { status: 'unauthorized' }, status: :unauthorized
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def http_status_for(status)
|
|
27
|
+
case status.to_s
|
|
28
|
+
when 'unhealthy' then :service_unavailable
|
|
29
|
+
else :ok
|
|
30
|
+
end
|
|
7
31
|
end
|
|
8
32
|
end
|
|
9
33
|
end
|
|
@@ -11,7 +11,11 @@ SolidQueueGuard.configure do |config|
|
|
|
11
11
|
|
|
12
12
|
config.failed_jobs_threshold = 20
|
|
13
13
|
config.stale_process_threshold = 5.minutes
|
|
14
|
+
config.health_cache_ttl = 15.seconds
|
|
15
|
+
config.scheduled_backlog_threshold = 100
|
|
14
16
|
|
|
15
17
|
# config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
|
|
16
|
-
# config.
|
|
18
|
+
# config.integrate_rails_health = true
|
|
19
|
+
# config.notify_with = [:rails_logger, :slack, :datadog, :webhook]
|
|
20
|
+
# config.metrics_backends = [:statsd, :prometheus, :opentelemetry]
|
|
17
21
|
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Checks
|
|
5
|
+
module Config
|
|
6
|
+
class TopologyRecommendationCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
recommendations = Recommendations::Topology.analyze
|
|
9
|
+
|
|
10
|
+
if recommendations.empty?
|
|
11
|
+
pass('topology_recommendation', 'No queue.yml topology changes recommended')
|
|
12
|
+
else
|
|
13
|
+
warn(
|
|
14
|
+
'topology_recommendation',
|
|
15
|
+
recommendations.join('; '),
|
|
16
|
+
suggestion: 'Review config/queue.yml worker and database pool settings',
|
|
17
|
+
metadata: { recommendations: recommendations }
|
|
18
|
+
)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -14,7 +14,8 @@ module SolidQueueGuard
|
|
|
14
14
|
Config::SchedulerConfigCheck,
|
|
15
15
|
Config::EnvFlagsCheck,
|
|
16
16
|
Config::ProcessHeartbeatConfigCheck,
|
|
17
|
-
Config::PumaColocatedCheck
|
|
17
|
+
Config::PumaColocatedCheck,
|
|
18
|
+
Config::TopologyRecommendationCheck
|
|
18
19
|
].freeze
|
|
19
20
|
|
|
20
21
|
RUNTIME_CHECKS = [
|
|
@@ -4,11 +4,7 @@ module SolidQueueGuard
|
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
6
|
class Base < Checks::Base
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def call
|
|
10
|
-
skip(check_id, NOT_IMPLEMENTED_MESSAGE)
|
|
11
|
-
end
|
|
7
|
+
include DatabaseSupport
|
|
12
8
|
|
|
13
9
|
private
|
|
14
10
|
|
|
@@ -3,7 +3,26 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class BlockedJobsCheck < Base
|
|
6
|
+
class BlockedJobsCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
blocked = SolidQueue::BlockedExecution.count
|
|
10
|
+
expired = SolidQueue::BlockedExecution.expired.count
|
|
11
|
+
|
|
12
|
+
if expired.positive?
|
|
13
|
+
warn(
|
|
14
|
+
check_id,
|
|
15
|
+
"#{expired} blocked job(s) have expired concurrency locks",
|
|
16
|
+
suggestion: 'Verify concurrency release maintenance is running'
|
|
17
|
+
)
|
|
18
|
+
elsif blocked.positive?
|
|
19
|
+
pass(check_id, "#{blocked} job(s) blocked by concurrency control")
|
|
20
|
+
else
|
|
21
|
+
pass(check_id, 'No blocked jobs')
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
7
26
|
end
|
|
8
27
|
end
|
|
9
28
|
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Checks
|
|
5
|
+
module Runtime
|
|
6
|
+
module DatabaseSupport
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def queue_database_available?
|
|
10
|
+
pool = SolidQueue::Record.connection_pool
|
|
11
|
+
return false if pool.nil?
|
|
12
|
+
|
|
13
|
+
pool.with_connection(&:active?)
|
|
14
|
+
rescue StandardError
|
|
15
|
+
false
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def with_queue_database
|
|
19
|
+
return skip(check_id, 'Queue database is not available') unless queue_database_available?
|
|
20
|
+
|
|
21
|
+
yield
|
|
22
|
+
rescue StandardError => e
|
|
23
|
+
skip(check_id, "Queue database query failed: #{e.class}")
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def config
|
|
27
|
+
SolidQueueGuard.config
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def lag_threshold_for(queue_name)
|
|
31
|
+
thresholds = config.queue_lag_thresholds
|
|
32
|
+
thresholds[queue_name.to_sym] || thresholds[queue_name] || thresholds[:default]
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -3,7 +3,30 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class DispatcherCheck < Base
|
|
6
|
+
class DispatcherCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
dispatchers = SolidQueue::Process.where(kind: 'Dispatcher')
|
|
10
|
+
due_count = SolidQueue::ScheduledExecution.due.count
|
|
11
|
+
|
|
12
|
+
if dispatchers.none? && due_count.positive?
|
|
13
|
+
failure(
|
|
14
|
+
check_id,
|
|
15
|
+
"#{due_count} scheduled job(s) are due but no dispatcher is running",
|
|
16
|
+
suggestion: 'Start a dispatcher process or verify bin/jobs is running'
|
|
17
|
+
)
|
|
18
|
+
elsif due_count > config.scheduled_backlog_threshold
|
|
19
|
+
warn(
|
|
20
|
+
check_id,
|
|
21
|
+
"#{due_count} scheduled executions are due (threshold: #{config.scheduled_backlog_threshold})",
|
|
22
|
+
suggestion: 'Verify the dispatcher is keeping up with scheduled work'
|
|
23
|
+
)
|
|
24
|
+
else
|
|
25
|
+
pass(check_id, "Dispatcher healthy (#{due_count} due scheduled executions)")
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
7
30
|
end
|
|
8
31
|
end
|
|
9
32
|
end
|
|
@@ -3,7 +3,27 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class FailedJobsCheck < Base
|
|
6
|
+
class FailedJobsCheck < Base
|
|
7
|
+
WINDOW = 1.hour
|
|
8
|
+
|
|
9
|
+
def call
|
|
10
|
+
with_queue_database do
|
|
11
|
+
count = SolidQueue::FailedExecution.where(created_at: WINDOW.ago..).count
|
|
12
|
+
threshold = config.failed_jobs_threshold
|
|
13
|
+
|
|
14
|
+
if count > threshold
|
|
15
|
+
warn(
|
|
16
|
+
check_id,
|
|
17
|
+
"#{count} failed job(s) in the last hour (threshold: #{threshold})",
|
|
18
|
+
suggestion: 'Review failed jobs in Mission Control or solid_queue_failed_executions',
|
|
19
|
+
metadata: { failed_jobs_last_hour: count }
|
|
20
|
+
)
|
|
21
|
+
else
|
|
22
|
+
pass(check_id, "#{count} failed job(s) in the last hour")
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
7
27
|
end
|
|
8
28
|
end
|
|
9
29
|
end
|
|
@@ -3,7 +3,26 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class FinishedJobsGrowthCheck < Base
|
|
6
|
+
class FinishedJobsGrowthCheck < Base
|
|
7
|
+
WINDOW = 24.hours
|
|
8
|
+
GROWTH_THRESHOLD = 10_000
|
|
9
|
+
|
|
10
|
+
def call
|
|
11
|
+
with_queue_database do
|
|
12
|
+
recent_finished = SolidQueue::Job.where(finished_at: WINDOW.ago..).count
|
|
13
|
+
|
|
14
|
+
if recent_finished > GROWTH_THRESHOLD
|
|
15
|
+
warn(
|
|
16
|
+
check_id,
|
|
17
|
+
"#{recent_finished} finished jobs in the last 24 hours",
|
|
18
|
+
suggestion: 'Ensure a recurring cleanup task runs (SolidQueue::Job.clear_finished_in_batches)'
|
|
19
|
+
)
|
|
20
|
+
else
|
|
21
|
+
pass(check_id, "#{recent_finished} finished jobs in the last 24 hours")
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
7
26
|
end
|
|
8
27
|
end
|
|
9
28
|
end
|
|
@@ -3,7 +3,24 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class OrphanedClaimsCheck < Base
|
|
6
|
+
class OrphanedClaimsCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
alive_process_ids = SolidQueue::Process.pluck(:id)
|
|
10
|
+
orphaned = SolidQueue::ClaimedExecution.where.not(process_id: alive_process_ids)
|
|
11
|
+
|
|
12
|
+
if orphaned.none?
|
|
13
|
+
pass(check_id, 'No orphaned claimed executions')
|
|
14
|
+
else
|
|
15
|
+
warn(
|
|
16
|
+
check_id,
|
|
17
|
+
"#{orphaned.count} claimed execution(s) belong to dead processes",
|
|
18
|
+
suggestion: 'Restart workers; Solid Queue maintenance should release orphaned claims'
|
|
19
|
+
)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
7
24
|
end
|
|
8
25
|
end
|
|
9
26
|
end
|
|
@@ -3,7 +3,28 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class PausedQueueLagCheck < Base
|
|
6
|
+
class PausedQueueLagCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
paused_queues = SolidQueue::Pause.pluck(:queue_name)
|
|
10
|
+
return pass(check_id, 'No paused queues') if paused_queues.empty?
|
|
11
|
+
|
|
12
|
+
growing = paused_queues.select do |queue_name|
|
|
13
|
+
SolidQueue::ReadyExecution.exists?(queue_name: queue_name)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
if growing.any?
|
|
17
|
+
warn(
|
|
18
|
+
check_id,
|
|
19
|
+
"Paused queue(s) still have ready jobs: #{growing.join(', ')}",
|
|
20
|
+
suggestion: 'Unpause queues or drain jobs before pausing'
|
|
21
|
+
)
|
|
22
|
+
else
|
|
23
|
+
pass(check_id, "Paused queues have no ready backlog: #{paused_queues.join(', ')}")
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
7
28
|
end
|
|
8
29
|
end
|
|
9
30
|
end
|
|
@@ -3,7 +3,39 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class PidfileCheck < Base
|
|
6
|
+
class PidfileCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
pidfile = ENV['SOLID_QUEUE_PIDFILE'] || default_pidfile
|
|
9
|
+
return pass(check_id, 'No Solid Queue pidfile configured') if pidfile.blank?
|
|
10
|
+
|
|
11
|
+
path = Pathname(pidfile)
|
|
12
|
+
unless path.exist?
|
|
13
|
+
return warn(check_id, "Pidfile not found at #{path}",
|
|
14
|
+
suggestion: 'Verify Solid Queue supervisor is running')
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
pid = path.read.strip.to_i
|
|
18
|
+
if process_alive?(pid)
|
|
19
|
+
pass(check_id, "Solid Queue supervisor pidfile present (pid #{pid})")
|
|
20
|
+
else
|
|
21
|
+
failure(check_id, "Pidfile exists but process #{pid} is not running",
|
|
22
|
+
suggestion: 'Restart the Solid Queue supervisor')
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def default_pidfile
|
|
29
|
+
rails_root.join('tmp/pids/solid_queue.pid').to_s
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def process_alive?(pid)
|
|
33
|
+
Process.getpgid(pid)
|
|
34
|
+
true
|
|
35
|
+
rescue Errno::ESRCH
|
|
36
|
+
false
|
|
37
|
+
end
|
|
38
|
+
end
|
|
7
39
|
end
|
|
8
40
|
end
|
|
9
41
|
end
|
|
@@ -3,7 +3,37 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class ProcessTopologyCheck < Base
|
|
6
|
+
class ProcessTopologyCheck < Base
|
|
7
|
+
EXPECTED_KINDS = %w[Supervisor Worker Dispatcher Scheduler].freeze
|
|
8
|
+
|
|
9
|
+
def call
|
|
10
|
+
with_queue_database do
|
|
11
|
+
kinds = SolidQueue::Process.distinct.pluck(:kind)
|
|
12
|
+
missing = EXPECTED_KINDS - kinds
|
|
13
|
+
|
|
14
|
+
if missing == EXPECTED_KINDS
|
|
15
|
+
warn(check_id, 'No Solid Queue processes are running',
|
|
16
|
+
suggestion: 'Start bin/jobs or the Solid Queue supervisor')
|
|
17
|
+
elsif missing.include?('Worker')
|
|
18
|
+
warn(check_id, 'No worker processes detected',
|
|
19
|
+
suggestion: 'Ensure workers are configured in queue.yml and running')
|
|
20
|
+
elsif missing.include?('Dispatcher') && scheduled_work_expected?
|
|
21
|
+
warn(check_id, 'No dispatcher process detected with scheduled work configured')
|
|
22
|
+
else
|
|
23
|
+
pass(check_id, "Solid Queue processes running: #{kinds.sort.join(', ')}")
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def scheduled_work_expected?
|
|
31
|
+
solid_queue_configuration.send(:recurring_tasks).any? ||
|
|
32
|
+
SolidQueue::ScheduledExecution.exists?
|
|
33
|
+
rescue StandardError
|
|
34
|
+
false
|
|
35
|
+
end
|
|
36
|
+
end
|
|
7
37
|
end
|
|
8
38
|
end
|
|
9
39
|
end
|
|
@@ -3,7 +3,36 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class QueueLagCheck < Base
|
|
6
|
+
class QueueLagCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
lags = SolidQueue::ReadyExecution.group(:queue_name).minimum(:created_at)
|
|
10
|
+
return pass(check_id, 'No jobs waiting in ready queues') if lags.empty?
|
|
11
|
+
|
|
12
|
+
worst = lags.map do |queue_name, oldest|
|
|
13
|
+
lag = Time.current - oldest
|
|
14
|
+
{ queue: queue_name, lag: lag, threshold: lag_threshold_for(queue_name) }
|
|
15
|
+
end.max_by { |entry| entry[:lag] }
|
|
16
|
+
|
|
17
|
+
if worst[:lag] > worst[:threshold]
|
|
18
|
+
failure(
|
|
19
|
+
check_id,
|
|
20
|
+
"#{worst[:queue]} queue lag is #{worst[:lag].to_i} seconds",
|
|
21
|
+
suggestion: "Investigate worker coverage and throughput for the #{worst[:queue]} queue",
|
|
22
|
+
metadata: { queue_lag_seconds: worst[:lag].to_i, queue_name: worst[:queue] }
|
|
23
|
+
)
|
|
24
|
+
elsif worst[:lag] > worst[:threshold] / 2
|
|
25
|
+
warn(
|
|
26
|
+
check_id,
|
|
27
|
+
"#{worst[:queue]} queue lag is #{worst[:lag].to_i} seconds",
|
|
28
|
+
metadata: { queue_lag_seconds: worst[:lag].to_i, queue_name: worst[:queue] }
|
|
29
|
+
)
|
|
30
|
+
else
|
|
31
|
+
pass(check_id, "Queue lag within thresholds (worst: #{worst[:queue]} at #{worst[:lag].to_i}s)")
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
7
36
|
end
|
|
8
37
|
end
|
|
9
38
|
end
|
|
@@ -3,7 +3,35 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class RecurringStaleCheck < Base
|
|
6
|
+
class RecurringStaleCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
tasks = SolidQueue::RecurringTask.static
|
|
10
|
+
return pass(check_id, 'No recurring tasks configured') if tasks.none?
|
|
11
|
+
|
|
12
|
+
stale_tasks = tasks.select do |task|
|
|
13
|
+
last_run = SolidQueue::RecurringExecution.where(task_key: task.key).maximum(:run_at)
|
|
14
|
+
last_run.nil? || last_run < expected_staleness.ago
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
if stale_tasks.any?
|
|
18
|
+
warn(
|
|
19
|
+
check_id,
|
|
20
|
+
"#{stale_tasks.size} recurring task(s) may be stale: #{stale_tasks.map(&:key).join(', ')}",
|
|
21
|
+
suggestion: 'Verify the scheduler process is running'
|
|
22
|
+
)
|
|
23
|
+
else
|
|
24
|
+
pass(check_id, 'Recurring tasks have recent executions')
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def expected_staleness
|
|
32
|
+
config.stale_process_threshold * 2
|
|
33
|
+
end
|
|
34
|
+
end
|
|
7
35
|
end
|
|
8
36
|
end
|
|
9
37
|
end
|
|
@@ -3,7 +3,24 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class ScheduledBacklogCheck < Base
|
|
6
|
+
class ScheduledBacklogCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
due_count = SolidQueue::ScheduledExecution.due.count
|
|
10
|
+
threshold = config.scheduled_backlog_threshold
|
|
11
|
+
|
|
12
|
+
if due_count > threshold
|
|
13
|
+
warn(
|
|
14
|
+
check_id,
|
|
15
|
+
"Scheduled backlog is #{due_count} (threshold: #{threshold})",
|
|
16
|
+
suggestion: 'Check dispatcher health and scheduled job volume'
|
|
17
|
+
)
|
|
18
|
+
else
|
|
19
|
+
pass(check_id, "Scheduled backlog is #{due_count}")
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
7
24
|
end
|
|
8
25
|
end
|
|
9
26
|
end
|
|
@@ -3,7 +3,25 @@
|
|
|
3
3
|
module SolidQueueGuard
|
|
4
4
|
module Checks
|
|
5
5
|
module Runtime
|
|
6
|
-
class StaleProcessCheck < Base
|
|
6
|
+
class StaleProcessCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
with_queue_database do
|
|
9
|
+
threshold = config.stale_process_threshold
|
|
10
|
+
stale = SolidQueue::Process.where(last_heartbeat_at: ...threshold.ago)
|
|
11
|
+
|
|
12
|
+
if stale.none?
|
|
13
|
+
pass(check_id, 'All Solid Queue processes have recent heartbeats')
|
|
14
|
+
else
|
|
15
|
+
failure(
|
|
16
|
+
check_id,
|
|
17
|
+
"#{stale.count} Solid Queue process(es) have stale heartbeats",
|
|
18
|
+
suggestion: 'Restart the Solid Queue supervisor and verify workers are running',
|
|
19
|
+
metadata: { stale_processes: stale.count }
|
|
20
|
+
)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
7
25
|
end
|
|
8
26
|
end
|
|
9
27
|
end
|
|
@@ -18,9 +18,12 @@ module SolidQueueGuard
|
|
|
18
18
|
strict = strict_flag?(argv) if strict.nil?
|
|
19
19
|
|
|
20
20
|
report = Runner.new(scope: scope).run
|
|
21
|
+
Notifier.deliver_all(report) if notify?(report)
|
|
22
|
+
Metrics::Exporter.export(report) if SolidQueueGuard.config.metrics_backends.any?
|
|
23
|
+
|
|
21
24
|
output = formatter_for(format).new(report).render
|
|
22
25
|
|
|
23
|
-
|
|
26
|
+
$stdout.puts(output) unless output.empty?
|
|
24
27
|
|
|
25
28
|
exit report.exit_code(strict: strict)
|
|
26
29
|
end
|
|
@@ -51,5 +54,11 @@ module SolidQueueGuard
|
|
|
51
54
|
ENV.fetch('SOLID_QUEUE_GUARD_SCOPE', 'config').to_sym
|
|
52
55
|
end
|
|
53
56
|
end
|
|
57
|
+
|
|
58
|
+
def notify?(report)
|
|
59
|
+
return false unless SolidQueueGuard.config.notify_with.any?
|
|
60
|
+
|
|
61
|
+
report.status != :healthy
|
|
62
|
+
end
|
|
54
63
|
end
|
|
55
64
|
end
|
|
@@ -11,7 +11,8 @@ module SolidQueueGuard
|
|
|
11
11
|
:health_cache_ttl,
|
|
12
12
|
:scheduled_backlog_threshold,
|
|
13
13
|
:integrate_rails_health,
|
|
14
|
-
:notify_with
|
|
14
|
+
:notify_with,
|
|
15
|
+
:metrics_backends
|
|
15
16
|
|
|
16
17
|
def initialize
|
|
17
18
|
@enabled = true
|
|
@@ -24,6 +25,7 @@ module SolidQueueGuard
|
|
|
24
25
|
@scheduled_backlog_threshold = 100
|
|
25
26
|
@integrate_rails_health = false
|
|
26
27
|
@notify_with = [:rails_logger]
|
|
28
|
+
@metrics_backends = []
|
|
27
29
|
end
|
|
28
30
|
|
|
29
31
|
def strict?
|
|
@@ -17,5 +17,17 @@ module SolidQueueGuard
|
|
|
17
17
|
end
|
|
18
18
|
end
|
|
19
19
|
end
|
|
20
|
+
|
|
21
|
+
initializer 'solid_queue_guard.rails_health' do
|
|
22
|
+
next unless SolidQueueGuard.config.integrate_rails_health
|
|
23
|
+
|
|
24
|
+
ActiveSupport.on_load(:action_controller_base) do
|
|
25
|
+
Rails::HealthController.class_eval do
|
|
26
|
+
def solid_queue_guard_status
|
|
27
|
+
SolidQueueGuard::Health::Cache.fetch[:status]
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
20
32
|
end
|
|
21
33
|
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Health
|
|
5
|
+
class Cache
|
|
6
|
+
def self.fetch
|
|
7
|
+
new.fetch
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def fetch
|
|
11
|
+
Rails.cache.fetch(cache_key, expires_in: SolidQueueGuard.config.health_cache_ttl) do
|
|
12
|
+
build_payload
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def cache_key
|
|
19
|
+
'solid_queue_guard/health'
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def build_payload
|
|
23
|
+
report = Runner.new(scope: :all).run
|
|
24
|
+
Payload.new(report).to_h
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Health
|
|
5
|
+
class Payload
|
|
6
|
+
def initialize(report)
|
|
7
|
+
@report = report
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def to_h
|
|
11
|
+
base = report.to_h.merge(
|
|
12
|
+
queue_lag_seconds: queue_lag_seconds,
|
|
13
|
+
failed_jobs_last_hour: failed_jobs_last_hour,
|
|
14
|
+
dead_processes: dead_processes
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
base[:recommendations] = topology_recommendations if topology_recommendations.any?
|
|
18
|
+
base
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
attr_reader :report
|
|
24
|
+
|
|
25
|
+
def queue_lag_seconds
|
|
26
|
+
metadata_value('queue_lag') { |metadata| metadata[:queue_lag_seconds] }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def failed_jobs_last_hour
|
|
30
|
+
metadata_value('failed_jobs') { |metadata| metadata[:failed_jobs_last_hour] } || 0
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def dead_processes
|
|
34
|
+
metadata_value('stale_process') { |metadata| metadata[:stale_processes] } || 0
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def metadata_value(check_id)
|
|
38
|
+
result = report.results.find { |entry| entry.id == check_id }
|
|
39
|
+
metadata = result&.metadata || {}
|
|
40
|
+
value = yield(metadata)
|
|
41
|
+
value unless value.nil?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def topology_recommendations
|
|
45
|
+
result = report.results.find { |entry| entry.id == 'topology_recommendation' }
|
|
46
|
+
result&.metadata&.fetch(:recommendations, []) || []
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Metrics
|
|
5
|
+
class Exporter
|
|
6
|
+
STATUS_VALUES = { healthy: 0, degraded: 1, unhealthy: 2 }.freeze
|
|
7
|
+
|
|
8
|
+
def self.export(report, backends: SolidQueueGuard.config.metrics_backends)
|
|
9
|
+
new(report, backends: backends).export
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def initialize(report, backends:)
|
|
13
|
+
@report = report
|
|
14
|
+
@backends = Array(backends)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def export
|
|
18
|
+
backends.each { |backend| export_backend(backend) }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
attr_reader :report, :backends
|
|
24
|
+
|
|
25
|
+
def export_backend(backend)
|
|
26
|
+
case backend.to_sym
|
|
27
|
+
when :statsd then Statsd.export(report)
|
|
28
|
+
when :prometheus then Prometheus.export(report)
|
|
29
|
+
when :opentelemetry then OpenTelemetry.export(report)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Metrics
|
|
5
|
+
module OpenTelemetry
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def export(report)
|
|
9
|
+
return unless defined?(::OpenTelemetry)
|
|
10
|
+
|
|
11
|
+
meter = ::OpenTelemetry.meter_provider.meter('solid_queue_guard')
|
|
12
|
+
gauge = meter.create_gauge('solid_queue.guard.overall_status', unit: 'status')
|
|
13
|
+
gauge.record(Exporter::STATUS_VALUES.fetch(report.status))
|
|
14
|
+
rescue StandardError
|
|
15
|
+
nil
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Metrics
|
|
5
|
+
module Prometheus
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def export(report)
|
|
9
|
+
path = ENV.fetch('SOLID_QUEUE_GUARD_PROMETHEUS_FILE', Rails.root.join('tmp/solid_queue_guard.prom'))
|
|
10
|
+
File.write(path, render(report))
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def render(report)
|
|
14
|
+
<<~PROM
|
|
15
|
+
# TYPE solid_queue_guard_overall_status gauge
|
|
16
|
+
solid_queue_guard_overall_status #{Exporter::STATUS_VALUES.fetch(report.status)}
|
|
17
|
+
PROM
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Metrics
|
|
5
|
+
module Statsd
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def export(report)
|
|
9
|
+
require 'socket'
|
|
10
|
+
host = ENV.fetch('SOLID_QUEUE_GUARD_STATSD_HOST', '127.0.0.1')
|
|
11
|
+
port = ENV.fetch('SOLID_QUEUE_GUARD_STATSD_PORT', 8125).to_i
|
|
12
|
+
socket = UDPSocket.new
|
|
13
|
+
socket.send(metric_line(report), 0, host, port)
|
|
14
|
+
ensure
|
|
15
|
+
socket&.close
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def metric_line(report)
|
|
19
|
+
"solid_queue.guard.overall_status:#{STATUS_VALUES.fetch(report.status)}|g"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
STATUS_VALUES = Exporter::STATUS_VALUES
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
class Notifier
|
|
5
|
+
class << self
|
|
6
|
+
def deliver_all(report, adapters: SolidQueueGuard.config.notify_with)
|
|
7
|
+
Array(adapters).each do |adapter|
|
|
8
|
+
deliver(adapter, report)
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def deliver(adapter, report)
|
|
13
|
+
case adapter.to_sym
|
|
14
|
+
when :slack then deliver_slack(report)
|
|
15
|
+
when :datadog then deliver_datadog(report)
|
|
16
|
+
when :webhook then deliver_webhook(report)
|
|
17
|
+
else deliver_rails_logger(report)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def post_json(url, payload, headers: {})
|
|
22
|
+
require 'net/http'
|
|
23
|
+
uri = URI(url)
|
|
24
|
+
request = Net::HTTP::Post.new(uri)
|
|
25
|
+
request['Content-Type'] = 'application/json'
|
|
26
|
+
headers.each { |key, value| request[key] = value }
|
|
27
|
+
request.body = JSON.generate(payload)
|
|
28
|
+
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
|
|
29
|
+
http.request(request)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def deliver_rails_logger(report)
|
|
36
|
+
Rails.logger.warn("[SolidQueueGuard] status=#{report.status} warnings=#{report.warnings.join('; ')}")
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def deliver_slack(report)
|
|
40
|
+
webhook_url = ENV.fetch('SOLID_QUEUE_GUARD_SLACK_WEBHOOK_URL', nil)
|
|
41
|
+
return if webhook_url.blank?
|
|
42
|
+
|
|
43
|
+
post_json(webhook_url, { text: "SolidQueueGuard status: *#{report.status}*\n#{report.warnings.join("\n")}" })
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def deliver_datadog(report)
|
|
47
|
+
api_key = ENV.fetch('DD_API_KEY', nil)
|
|
48
|
+
return if api_key.blank?
|
|
49
|
+
|
|
50
|
+
alert_type = case report.status
|
|
51
|
+
when :unhealthy then 'error'
|
|
52
|
+
when :degraded then 'warning'
|
|
53
|
+
else 'info'
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
post_json(
|
|
57
|
+
'https://api.datadoghq.com/api/v1/events',
|
|
58
|
+
{
|
|
59
|
+
title: 'SolidQueueGuard alert',
|
|
60
|
+
text: report.warnings.join("\n"),
|
|
61
|
+
alert_type: alert_type,
|
|
62
|
+
tags: ["status:#{report.status}"]
|
|
63
|
+
},
|
|
64
|
+
headers: { 'DD-API-KEY' => api_key }
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def deliver_webhook(report)
|
|
69
|
+
webhook_url = ENV.fetch('SOLID_QUEUE_GUARD_WEBHOOK_URL', nil)
|
|
70
|
+
return if webhook_url.blank?
|
|
71
|
+
|
|
72
|
+
post_json(webhook_url, report.to_h)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -4,14 +4,9 @@ module SolidQueueGuard
|
|
|
4
4
|
module Notifiers
|
|
5
5
|
# @api private
|
|
6
6
|
class Base
|
|
7
|
-
def self.deliver(
|
|
8
|
-
|
|
7
|
+
def self.deliver(report)
|
|
8
|
+
Notifier.deliver(:rails_logger, report)
|
|
9
9
|
end
|
|
10
10
|
end
|
|
11
|
-
|
|
12
|
-
class RailsLogger < Base; end
|
|
13
|
-
class Slack < Base; end
|
|
14
|
-
class Datadog < Base; end
|
|
15
|
-
class Webhook < Base; end
|
|
16
11
|
end
|
|
17
12
|
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Recommendations
|
|
5
|
+
class Topology
|
|
6
|
+
def self.analyze(configuration: SolidQueue::Configuration.new)
|
|
7
|
+
new(configuration: configuration).analyze
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def initialize(configuration:)
|
|
11
|
+
@configuration = configuration
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def analyze
|
|
15
|
+
recommendations = []
|
|
16
|
+
recommendations.concat(worker_recommendations)
|
|
17
|
+
recommendations.concat(pool_recommendations)
|
|
18
|
+
recommendations.uniq
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
attr_reader :configuration
|
|
24
|
+
|
|
25
|
+
def worker_recommendations
|
|
26
|
+
worker_queues = QueueCoverage.worker_queues_from_configuration(configuration)
|
|
27
|
+
uncovered = uncovered_queues(worker_queues)
|
|
28
|
+
return [] if uncovered.empty?
|
|
29
|
+
|
|
30
|
+
["Add worker coverage for: #{uncovered.join(', ')}"]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def pool_recommendations
|
|
34
|
+
required_threads = configuration.estimated_number_of_threads
|
|
35
|
+
pool_size = SolidQueue::Record.connection_pool&.size
|
|
36
|
+
return [] if pool_size.nil? || required_threads <= pool_size
|
|
37
|
+
|
|
38
|
+
["Increase queue DB pool to at least #{required_threads + 2} (currently #{pool_size})"]
|
|
39
|
+
rescue StandardError
|
|
40
|
+
[]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def uncovered_queues(assigned_queues)
|
|
44
|
+
observed = observed_queues
|
|
45
|
+
(observed - assigned_queues).sort
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def observed_queues
|
|
49
|
+
return [] unless database_available?
|
|
50
|
+
|
|
51
|
+
SolidQueue::Job.distinct.pluck(:queue_name)
|
|
52
|
+
rescue StandardError
|
|
53
|
+
[]
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def database_available?
|
|
57
|
+
SolidQueue::Record.connection_pool&.with_connection(&:active?)
|
|
58
|
+
rescue StandardError
|
|
59
|
+
false
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -17,7 +17,7 @@ namespace :solid_queue_guard do
|
|
|
17
17
|
|
|
18
18
|
desc 'Print machine-readable health status'
|
|
19
19
|
task health: :environment do
|
|
20
|
-
SolidQueueGuard::CLI.run!(scope: :
|
|
20
|
+
SolidQueueGuard::CLI.run!(scope: :all, format: :json, strict: cli_strict)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
desc 'Print full diagnostic report. Usage: report[json]'
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: solid_queue_guard
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rafael Pissardo
|
|
@@ -290,10 +290,12 @@ files:
|
|
|
290
290
|
- lib/solid_queue_guard/checks/config/queue_schema_check.rb
|
|
291
291
|
- lib/solid_queue_guard/checks/config/scheduler_config_check.rb
|
|
292
292
|
- lib/solid_queue_guard/checks/config/thread_pool_check.rb
|
|
293
|
+
- lib/solid_queue_guard/checks/config/topology_recommendation_check.rb
|
|
293
294
|
- lib/solid_queue_guard/checks/config/worker_coverage_check.rb
|
|
294
295
|
- lib/solid_queue_guard/checks/registry.rb
|
|
295
296
|
- lib/solid_queue_guard/checks/runtime/base.rb
|
|
296
297
|
- lib/solid_queue_guard/checks/runtime/blocked_jobs_check.rb
|
|
298
|
+
- lib/solid_queue_guard/checks/runtime/database_support.rb
|
|
297
299
|
- lib/solid_queue_guard/checks/runtime/dispatcher_check.rb
|
|
298
300
|
- lib/solid_queue_guard/checks/runtime/failed_jobs_check.rb
|
|
299
301
|
- lib/solid_queue_guard/checks/runtime/finished_jobs_growth_check.rb
|
|
@@ -310,8 +312,16 @@ files:
|
|
|
310
312
|
- lib/solid_queue_guard/engine.rb
|
|
311
313
|
- lib/solid_queue_guard/formatters/json.rb
|
|
312
314
|
- lib/solid_queue_guard/formatters/terminal.rb
|
|
315
|
+
- lib/solid_queue_guard/health/cache.rb
|
|
316
|
+
- lib/solid_queue_guard/health/payload.rb
|
|
317
|
+
- lib/solid_queue_guard/metrics/exporter.rb
|
|
318
|
+
- lib/solid_queue_guard/metrics/open_telemetry.rb
|
|
319
|
+
- lib/solid_queue_guard/metrics/prometheus.rb
|
|
320
|
+
- lib/solid_queue_guard/metrics/statsd.rb
|
|
321
|
+
- lib/solid_queue_guard/notifier.rb
|
|
313
322
|
- lib/solid_queue_guard/notifiers/base.rb
|
|
314
323
|
- lib/solid_queue_guard/queue_coverage.rb
|
|
324
|
+
- lib/solid_queue_guard/recommendations/topology.rb
|
|
315
325
|
- lib/solid_queue_guard/report.rb
|
|
316
326
|
- lib/solid_queue_guard/runner.rb
|
|
317
327
|
- lib/solid_queue_guard/tasks.rb
|