solid_queue_guard 0.5.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +12 -0
- data/.rubocop.yml +44 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +88 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +308 -0
- data/README.md +215 -11
- data/app/controllers/solid_queue_guard/health_controller.rb +1 -4
- data/lib/generators/solid_queue_guard/install/USAGE.ci +9 -0
- data/lib/generators/solid_queue_guard/install/ci_generator.rb +19 -0
- data/lib/generators/solid_queue_guard/install/templates/solid_queue_guard.rb +6 -0
- data/lib/generators/solid_queue_guard/install/templates/solid_queue_guard.yml +25 -0
- data/lib/solid_queue_guard/checks/base.rb +16 -0
- data/lib/solid_queue_guard/checks/config/async_supervisor_config_check.rb +25 -0
- data/lib/solid_queue_guard/checks/config/puma_colocated_check.rb +5 -11
- data/lib/solid_queue_guard/checks/config/queue_schema_check.rb +80 -7
- data/lib/solid_queue_guard/checks/registry.rb +8 -2
- data/lib/solid_queue_guard/checks/runtime/database_support.rb +5 -2
- data/lib/solid_queue_guard/checks/runtime/dispatcher_check.rb +4 -2
- data/lib/solid_queue_guard/checks/runtime/failed_jobs_check.rb +1 -1
- data/lib/solid_queue_guard/checks/runtime/puma_plugin_runtime_check.rb +39 -0
- data/lib/solid_queue_guard/checks/runtime/recurring_stale_check.rb +1 -1
- data/lib/solid_queue_guard/checks/runtime/scheduled_backlog_check.rb +1 -1
- data/lib/solid_queue_guard/checks/runtime/stale_process_check.rb +1 -1
- data/lib/solid_queue_guard/configuration.rb +34 -1
- data/lib/solid_queue_guard/http_status_policy.rb +35 -0
- data/lib/solid_queue_guard/puma_plugin_support.rb +35 -0
- data/lib/solid_queue_guard/recommendations/topology.rb +13 -0
- data/lib/solid_queue_guard/runner.rb +10 -1
- data/lib/solid_queue_guard/schema/solid_queue_tables.rb +126 -0
- data/lib/solid_queue_guard/version.rb +1 -1
- data/lib/solid_queue_guard.rb +1 -1
- data/solid_queue_guard.gemspec +54 -0
- metadata +16 -1
data/README.md
CHANGED
|
@@ -49,7 +49,7 @@ Checks:
|
|
|
49
49
|
✅ Active Job adapter is :solid_queue
|
|
50
50
|
✅ Queue database configured in database.yml
|
|
51
51
|
✅ Solid Queue connects_to queue database (pool: 10)
|
|
52
|
-
✅
|
|
52
|
+
✅ Solid Queue schema tables present
|
|
53
53
|
❌ Worker threads: 10, queue DB pool: 5
|
|
54
54
|
⚠️ No workers configured for "mailers" queue
|
|
55
55
|
⚠️ recurring.yml exists but scheduler may not run
|
|
@@ -67,7 +67,7 @@ One command. Actionable output. No Datadog required to get started.
|
|
|
67
67
|
|
|
68
68
|
## What it checks
|
|
69
69
|
|
|
70
|
-
###
|
|
70
|
+
### Configuration doctor
|
|
71
71
|
|
|
72
72
|
Runs locally, in CI, or pre-deploy. **No extra infrastructure.**
|
|
73
73
|
|
|
@@ -76,15 +76,17 @@ Runs locally, in CI, or pre-deploy. **No extra infrastructure.**
|
|
|
76
76
|
| **Adapter** | Active Job not set to `:solid_queue` |
|
|
77
77
|
| **Queue database** | Missing `queue` entry in `database.yml` |
|
|
78
78
|
| **connects_to** | Solid Queue pointing at the wrong DB / pool |
|
|
79
|
-
| **Queue schema** | Missing
|
|
79
|
+
| **Queue schema** | Missing Solid Queue tables in schema files or queue database |
|
|
80
80
|
| **Thread pool** | `threads` > available queue DB connections |
|
|
81
81
|
| **Worker coverage** | Queues with no worker assigned |
|
|
82
82
|
| **Scheduler** | `recurring.yml` tasks without a scheduler |
|
|
83
83
|
| **Env flags** | `SOLID_QUEUE_SKIP_RECURRING=true` in production |
|
|
84
84
|
| **Heartbeats** | Default thresholds that may not fit your deploy |
|
|
85
85
|
| **Puma plugin** | Solid Queue co-located with web in production |
|
|
86
|
+
| **Async supervisor** | `async` mode with thread/pool sizing risks |
|
|
87
|
+
| **Topology** | `queue.yml` worker and pool recommendations |
|
|
86
88
|
|
|
87
|
-
###
|
|
89
|
+
### Runtime guards
|
|
88
90
|
|
|
89
91
|
| Check | Catches |
|
|
90
92
|
| ----- | ------- |
|
|
@@ -93,6 +95,7 @@ Runs locally, in CI, or pre-deploy. **No extra infrastructure.**
|
|
|
93
95
|
| **Dispatcher health** | Scheduled jobs never becoming ready |
|
|
94
96
|
| **Blocked jobs** | Concurrency control silently holding jobs |
|
|
95
97
|
| **Recurring staleness** | Cron tasks that stopped firing |
|
|
98
|
+
| **Puma plugin runtime** | Plugin enabled but no active Solid Queue processes |
|
|
96
99
|
| **HTTP `/health`** | Kamal, ECS, K8s, UptimeRobot integration |
|
|
97
100
|
|
|
98
101
|
---
|
|
@@ -104,6 +107,7 @@ Runs locally, in CI, or pre-deploy. **No extra infrastructure.**
|
|
|
104
107
|
```bash
|
|
105
108
|
bundle add solid_queue_guard
|
|
106
109
|
bin/rails solid_queue_guard:install
|
|
110
|
+
bin/rails generate solid_queue_guard:install:ci # optional GitHub Actions workflow
|
|
107
111
|
```
|
|
108
112
|
|
|
109
113
|
### Run the doctor
|
|
@@ -138,6 +142,12 @@ Perfect for deploy pipelines:
|
|
|
138
142
|
run: SOLID_QUEUE_GUARD_STRICT=1 bin/rails solid_queue_guard:doctor
|
|
139
143
|
```
|
|
140
144
|
|
|
145
|
+
Or generate a workflow:
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
bin/rails generate solid_queue_guard:install:ci
|
|
149
|
+
```
|
|
150
|
+
|
|
141
151
|
### HTTP health
|
|
142
152
|
|
|
143
153
|
```ruby
|
|
@@ -157,10 +167,164 @@ config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
|
|
|
157
167
|
# curl -H "X-Solid-Queue-Guard-Token: $TOKEN" ...
|
|
158
168
|
```
|
|
159
169
|
|
|
170
|
+
HTTP status policy (for Kamal, ECS, load balancers):
|
|
171
|
+
|
|
172
|
+
```ruby
|
|
173
|
+
config.degraded_http_status = 207 # or :ok (200), 503, etc.
|
|
174
|
+
config.unhealthy_http_status = 503 # default
|
|
175
|
+
```
|
|
176
|
+
|
|
160
177
|
Works with **Kamal**, **Heroku**, **Fly.io**, **ECS/Fargate**, **Kubernetes**, **Better Stack**, **UptimeRobot**.
|
|
161
178
|
|
|
162
179
|
---
|
|
163
180
|
|
|
181
|
+
## How to use it
|
|
182
|
+
|
|
183
|
+
Solid Queue does not show up in Rails `/up`. **solid_queue_guard** gives you three operational surfaces:
|
|
184
|
+
|
|
185
|
+
| Surface | Command / URL | Best for |
|
|
186
|
+
| ------- | ------------- | -------- |
|
|
187
|
+
| **Doctor** | `bin/rails solid_queue_guard:doctor` | Local pre-deploy, config review |
|
|
188
|
+
| **CI gate** | `SOLID_QUEUE_GUARD_STRICT=1 bin/rails solid_queue_guard:doctor` | Block merges with broken queue config |
|
|
189
|
+
| **HTTP health** | `GET /solid_queue_guard/health` | Production uptime monitors (Kamal, ECS, UptimeRobot) |
|
|
190
|
+
|
|
191
|
+
**Mission Control** shows what is happening. **solid_queue_guard** warns what is dangerous. Use both.
|
|
192
|
+
|
|
193
|
+
### Local and pre-deploy
|
|
194
|
+
|
|
195
|
+
Run before changing `queue.yml`, `database.yml`, or recurring tasks:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
bin/rails solid_queue_guard:doctor # config checks (default scope)
|
|
199
|
+
bin/rails solid_queue_guard:report # config + runtime when the queue DB is available
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
By default the install generator sets `config.enabled = Rails.env.production?`, so in **development** checks are skipped unless you enable them:
|
|
203
|
+
|
|
204
|
+
```ruby
|
|
205
|
+
SolidQueueGuard.configure { |c| c.enabled = true }
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Or run with production config locally when validating deploy readiness.
|
|
209
|
+
|
|
210
|
+
### CI pipelines
|
|
211
|
+
|
|
212
|
+
Validate **configuration** before deploy. Process/runtime checks usually **skip** in CI because there is no `bin/jobs` supervisor on the runner.
|
|
213
|
+
|
|
214
|
+
```yaml
|
|
215
|
+
- name: Solid Queue production readiness
|
|
216
|
+
run: SOLID_QUEUE_GUARD_STRICT=1 bin/rails solid_queue_guard:doctor
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
`STRICT=1` turns warnings into exit code `1`, so the pipeline fails on misconfigured pools, missing worker coverage, or missing schema tables — not only hard failures.
|
|
220
|
+
|
|
221
|
+
Generate a starter workflow:
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
bin/rails generate solid_queue_guard:install:ci
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
### Production monitoring
|
|
228
|
+
|
|
229
|
+
Mount the engine and point your load balancer or Kamal health check at `/solid_queue_guard/health`:
|
|
230
|
+
|
|
231
|
+
```ruby
|
|
232
|
+
# config/routes.rb
|
|
233
|
+
mount SolidQueueGuard::Engine, at: "/solid_queue_guard"
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Runtime checks matter here: queue lag, stale heartbeats, dispatcher health, and process topology reflect **live** Solid Queue state.
|
|
237
|
+
|
|
238
|
+
Optional hardening:
|
|
239
|
+
|
|
240
|
+
```ruby
|
|
241
|
+
config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
|
|
242
|
+
config.health_cache_ttl = 15.seconds
|
|
243
|
+
config.degraded_http_status = 207 # or :ok (200), 503, etc.
|
|
244
|
+
config.notify_with = [:rails_logger, :slack]
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
### Typical flow
|
|
248
|
+
|
|
249
|
+
```text
|
|
250
|
+
Developer CI Production
|
|
251
|
+
│ │ │
|
|
252
|
+
│ doctor │ doctor --strict │ GET /health (every 30–60s)
|
|
253
|
+
▼ ▼ ▼
|
|
254
|
+
"pool wrong?" block bad deploy alert: worker dead
|
|
255
|
+
"schema ok?" before merge alert: queue lag
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
## Runtime process checks
|
|
261
|
+
|
|
262
|
+
Runtime checks query the **queue database** (via `SolidQueue::Record`). If `connects_to` is missing or the DB is unreachable, they **skip** — they do not pass silently as healthy.
|
|
263
|
+
|
|
264
|
+
### `process_topology` — are the expected roles present?
|
|
265
|
+
|
|
266
|
+
Reads distinct `kind` values from `solid_queue_processes`:
|
|
267
|
+
|
|
268
|
+
| Kind | Role |
|
|
269
|
+
| ---- | ---- |
|
|
270
|
+
| `Supervisor` | Parent process (`bin/jobs`) |
|
|
271
|
+
| `Worker` | Consumes ready jobs |
|
|
272
|
+
| `Dispatcher` | Moves scheduled jobs to ready |
|
|
273
|
+
| `Scheduler` | Runs recurring tasks |
|
|
274
|
+
|
|
275
|
+
| Result | Meaning |
|
|
276
|
+
| ------ | ------- |
|
|
277
|
+
| ⚠️ No processes | Nothing registered — supervisor not running |
|
|
278
|
+
| ⚠️ No Worker | Jobs will not be processed |
|
|
279
|
+
| ⚠️ No Dispatcher | Scheduled work may stall (when recurring/scheduled jobs exist) |
|
|
280
|
+
| ✅ Pass | Expected kinds are present |
|
|
281
|
+
|
|
282
|
+
This check looks at **presence of records**, not heartbeat freshness.
|
|
283
|
+
|
|
284
|
+
### `stale_process` — are heartbeats fresh?
|
|
285
|
+
|
|
286
|
+
Finds processes where `last_heartbeat_at` is older than `stale_process_threshold` (default **5 minutes**):
|
|
287
|
+
|
|
288
|
+
| Result | Meaning |
|
|
289
|
+
| ------ | ------- |
|
|
290
|
+
| ✅ Pass | All processes reported recently |
|
|
291
|
+
| ❌ Fail | One or more workers/dispatchers look dead or stuck |
|
|
292
|
+
|
|
293
|
+
Use this in production health to catch workers that died after deploy.
|
|
294
|
+
|
|
295
|
+
### `pidfile` — optional supervisor liveness
|
|
296
|
+
|
|
297
|
+
When `tmp/pids/solid_queue.pid` (or `SOLID_QUEUE_PIDFILE`) exists, verifies the PID is alive. Often **warns in development** where `bin/jobs` is not running. Disable if you do not use pidfiles:
|
|
298
|
+
|
|
299
|
+
```ruby
|
|
300
|
+
config.disabled_checks = [:pidfile]
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
### `puma_plugin_runtime` — Solid Queue inside Puma
|
|
304
|
+
|
|
305
|
+
When `plugin :solid_queue` is in `config/puma.rb`, verifies active processes with recent heartbeats exist. **Skips** when the Puma plugin is not enabled.
|
|
306
|
+
|
|
307
|
+
### Queue schema detection
|
|
308
|
+
|
|
309
|
+
`QueueSchemaCheck` does **not** require `db/queue_schema.rb`. It validates that all tables for your installed **solid_queue** version exist in any of:
|
|
310
|
+
|
|
311
|
+
- `db/queue_schema.rb`
|
|
312
|
+
- `db/schema.rb`
|
|
313
|
+
- `db/structure.sql`
|
|
314
|
+
- the connected queue database
|
|
315
|
+
|
|
316
|
+
Apps that keep Solid Queue tables only in `structure.sql` (common in Revelo-style repos) pass correctly.
|
|
317
|
+
|
|
318
|
+
### Where each check type applies
|
|
319
|
+
|
|
320
|
+
| Context | Config checks | Process / runtime checks |
|
|
321
|
+
| ------- | ------------- | ------------------------ |
|
|
322
|
+
| Local `doctor` | ✅ Primary value | ⚠️ Partial without `bin/jobs` |
|
|
323
|
+
| CI `--strict` | ✅ Primary value | ⏭️ Usually skip |
|
|
324
|
+
| Production `/health` | ✅ When DB up | ✅ Primary value |
|
|
325
|
+
|
|
326
|
+
---
|
|
327
|
+
|
|
164
328
|
## Configuration
|
|
165
329
|
|
|
166
330
|
```ruby
|
|
@@ -177,13 +341,46 @@ SolidQueueGuard.configure do |config|
|
|
|
177
341
|
config.failed_jobs_threshold = 20
|
|
178
342
|
config.stale_process_threshold = 5.minutes
|
|
179
343
|
|
|
180
|
-
#
|
|
181
|
-
|
|
344
|
+
# Per-check overrides (v0.6+)
|
|
345
|
+
config.disabled_checks = [:pidfile]
|
|
346
|
+
config.checks.queue_lag = { threshold: 10.minutes }
|
|
347
|
+
config.checks.failed_jobs = { threshold: 5, enabled: true }
|
|
348
|
+
|
|
349
|
+
# HTTP status policy (v0.8+)
|
|
350
|
+
# config.degraded_http_status = 207
|
|
351
|
+
# config.unhealthy_http_status = 503
|
|
352
|
+
|
|
353
|
+
# config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
|
|
354
|
+
# config.integrate_rails_health = true
|
|
355
|
+
# config.notify_with = [:rails_logger, :slack, :datadog, :webhook]
|
|
356
|
+
# config.metrics_backends = [:statsd, :prometheus, :opentelemetry]
|
|
182
357
|
end
|
|
183
358
|
```
|
|
184
359
|
|
|
185
360
|
---
|
|
186
361
|
|
|
362
|
+
## Public API (v1.0+)
|
|
363
|
+
|
|
364
|
+
The following surface is **stable** until `2.0` and follows [semantic versioning](https://semver.org/):
|
|
365
|
+
|
|
366
|
+
| API | Description |
|
|
367
|
+
| --- | ----------- |
|
|
368
|
+
| `SolidQueueGuard.configure` | Block-style configuration |
|
|
369
|
+
| `SolidQueueGuard.config` | Current configuration object |
|
|
370
|
+
| `SolidQueueGuard.enabled?` | Whether checks run |
|
|
371
|
+
| `solid_queue_guard:doctor` | Config readiness report |
|
|
372
|
+
| `solid_queue_guard:health` | Runtime health report |
|
|
373
|
+
| `solid_queue_guard:report` | Full diagnostic report |
|
|
374
|
+
| `solid_queue_guard:install` | Initializer generator |
|
|
375
|
+
| `solid_queue_guard:install:ci` | GitHub Actions workflow generator |
|
|
376
|
+
| `mount SolidQueueGuard::Engine` | HTTP health endpoint |
|
|
377
|
+
|
|
378
|
+
Configuration attributes, rake tasks, and health JSON shape are public. Internal check classes and registry are `@api private`.
|
|
379
|
+
|
|
380
|
+
Breaking changes ship only in major versions (`2.0+`). Deprecations warn one minor version ahead.
|
|
381
|
+
|
|
382
|
+
---
|
|
383
|
+
|
|
187
384
|
## solid_queue_guard vs Mission Control
|
|
188
385
|
|
|
189
386
|
| | [Mission Control — Jobs](https://github.com/rails/mission_control-jobs) | solid_queue_guard |
|
|
@@ -192,9 +389,9 @@ end
|
|
|
192
389
|
| **UI** | Dashboard | CLI + JSON + health endpoint |
|
|
193
390
|
| **Retry / discard** | Yes | No (use Mission Control) |
|
|
194
391
|
| **Config doctor** | No | Yes |
|
|
195
|
-
| **Queue lag alerts** | No | Yes
|
|
392
|
+
| **Queue lag alerts** | No | Yes |
|
|
196
393
|
| **Pre-deploy checks** | No | Yes |
|
|
197
|
-
| **Recurring job guard** | Manual inspection | Automatic
|
|
394
|
+
| **Recurring job guard** | Manual inspection | Automatic |
|
|
198
395
|
|
|
199
396
|
**Use both.** They solve different problems.
|
|
200
397
|
|
|
@@ -202,13 +399,17 @@ end
|
|
|
202
399
|
|
|
203
400
|
## Roadmap
|
|
204
401
|
|
|
205
|
-
| Version | Ships |
|
|
206
|
-
| ------- | ----- |
|
|
402
|
+
| Version | Ships | Status |
|
|
403
|
+
| ------- | ----- | ------ |
|
|
207
404
|
| **v0.1** | `doctor` — config checks, CI integration, install generator | ✅ Released |
|
|
208
405
|
| **v0.2** | Runtime health, queue lag, dispatcher/blocked jobs, HTTP endpoint | ✅ Released |
|
|
209
406
|
| **v0.3** | Slack, Datadog, webhook notifications | ✅ Released |
|
|
210
407
|
| **v0.4** | StatsD, Prometheus, OpenTelemetry metrics | ✅ Released |
|
|
211
408
|
| **v0.5** | Auto-recommendations for `queue.yml` topology | ✅ Released |
|
|
409
|
+
| **v0.6** | Per-check configuration (`disabled_checks`, `config.checks`) | ✅ Released |
|
|
410
|
+
| **v0.7** | Puma plugin runtime check, async supervisor awareness | ✅ Released |
|
|
411
|
+
| **v0.8** | HTTP status policy, `install:ci` generator | ✅ Released |
|
|
412
|
+
| **v1.0** | Stable public API, strict semver | ✅ Released |
|
|
212
413
|
|
|
213
414
|
---
|
|
214
415
|
|
|
@@ -216,8 +417,9 @@ end
|
|
|
216
417
|
|
|
217
418
|
| Gem version | Ruby | Rails |
|
|
218
419
|
| ----------- | ---- | ----- |
|
|
219
|
-
| 0.
|
|
420
|
+
| 1.0.x | 3.1+ | 7.1, 7.2, 8.0 |
|
|
220
421
|
| 0.5.x | 3.1+ | 7.1, 7.2, 8.0 |
|
|
422
|
+
| 0.1.x | 3.1+ | 7.1, 7.2, 8.0 |
|
|
221
423
|
|
|
222
424
|
## Requirements
|
|
223
425
|
|
|
@@ -240,6 +442,8 @@ bundle exec appraisal install
|
|
|
240
442
|
bundle exec appraisal rake test
|
|
241
443
|
```
|
|
242
444
|
|
|
445
|
+
Release a new version by pushing a `v*` tag after CI passes (Trusted Publishing on RubyGems).
|
|
446
|
+
|
|
243
447
|
---
|
|
244
448
|
|
|
245
449
|
## Contributing
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/generators/base'
|
|
4
|
+
|
|
5
|
+
module SolidQueueGuard
|
|
6
|
+
module Generators
|
|
7
|
+
module Install
|
|
8
|
+
class CiGenerator < Rails::Generators::Base
|
|
9
|
+
source_root File.expand_path('templates', __dir__)
|
|
10
|
+
|
|
11
|
+
desc 'Adds a GitHub Actions workflow that runs solid_queue_guard:doctor in CI'
|
|
12
|
+
|
|
13
|
+
def copy_workflow
|
|
14
|
+
template 'solid_queue_guard.yml', '.github/workflows/solid_queue_guard.yml'
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -14,6 +14,12 @@ SolidQueueGuard.configure do |config|
|
|
|
14
14
|
config.health_cache_ttl = 15.seconds
|
|
15
15
|
config.scheduled_backlog_threshold = 100
|
|
16
16
|
|
|
17
|
+
# config.disabled_checks = [:pidfile]
|
|
18
|
+
# config.checks.queue_lag = { threshold: 10.minutes }
|
|
19
|
+
# config.checks.failed_jobs = { threshold: 5, enabled: true }
|
|
20
|
+
# config.degraded_http_status = 207
|
|
21
|
+
# config.unhealthy_http_status = 503
|
|
22
|
+
|
|
17
23
|
# config.health_token = ENV["SOLID_QUEUE_GUARD_TOKEN"]
|
|
18
24
|
# config.integrate_rails_health = true
|
|
19
25
|
# config.notify_with = [:rails_logger, :slack, :datadog, :webhook]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
name: Solid Queue Guard
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
doctor:
|
|
10
|
+
name: Solid Queue production readiness
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- uses: ruby/setup-ruby@v1
|
|
17
|
+
with:
|
|
18
|
+
ruby-version: .ruby-version
|
|
19
|
+
bundler-cache: true
|
|
20
|
+
|
|
21
|
+
- name: Prepare test database
|
|
22
|
+
run: bin/rails db:test:prepare
|
|
23
|
+
|
|
24
|
+
- name: Run Solid Queue Guard doctor
|
|
25
|
+
run: SOLID_QUEUE_GUARD_STRICT=1 bin/rails solid_queue_guard:doctor
|
|
@@ -8,6 +8,10 @@ module SolidQueueGuard
|
|
|
8
8
|
new(**options).call
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
+
def self.check_id
|
|
12
|
+
name.demodulize.underscore.delete_suffix('_check')
|
|
13
|
+
end
|
|
14
|
+
|
|
11
15
|
def initialize(**options)
|
|
12
16
|
@options = options
|
|
13
17
|
end
|
|
@@ -47,6 +51,18 @@ module SolidQueueGuard
|
|
|
47
51
|
def solid_queue_configuration
|
|
48
52
|
@solid_queue_configuration ||= SolidQueue::Configuration.new
|
|
49
53
|
end
|
|
54
|
+
|
|
55
|
+
def check_id
|
|
56
|
+
self.class.check_id
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def guard_config
|
|
60
|
+
SolidQueueGuard.config
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def check_setting(key, default = nil)
|
|
64
|
+
guard_config.check_setting(check_id, key, default)
|
|
65
|
+
end
|
|
50
66
|
end
|
|
51
67
|
end
|
|
52
68
|
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidQueueGuard
|
|
4
|
+
module Checks
|
|
5
|
+
module Config
|
|
6
|
+
class AsyncSupervisorConfigCheck < Base
|
|
7
|
+
def call
|
|
8
|
+
if PumaPluginSupport.async_supervisor_mode?
|
|
9
|
+
warn(
|
|
10
|
+
check_id,
|
|
11
|
+
'Solid Queue supervisor is running in async mode',
|
|
12
|
+
suggestion: [
|
|
13
|
+
'Review thread and database pool sizing;',
|
|
14
|
+
'the processes option in queue.yml is ignored in async mode'
|
|
15
|
+
].join(' '),
|
|
16
|
+
metadata: { supervisor_mode: 'async' }
|
|
17
|
+
)
|
|
18
|
+
else
|
|
19
|
+
pass(check_id, 'Solid Queue supervisor is running in fork mode')
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -4,27 +4,21 @@ module SolidQueueGuard
|
|
|
4
4
|
module Checks
|
|
5
5
|
module Config
|
|
6
6
|
class PumaColocatedCheck < Base
|
|
7
|
-
PUMA_PLUGIN_PATTERN = /plugin\s+:?solid_queue/
|
|
8
|
-
|
|
9
7
|
def call
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
return pass('puma_colocated', 'No config/puma.rb found') unless puma_config.exist?
|
|
13
|
-
|
|
14
|
-
content = puma_config.read
|
|
8
|
+
return pass(check_id, 'No config/puma.rb found') unless PumaPluginSupport.puma_config_path.exist?
|
|
15
9
|
|
|
16
|
-
if
|
|
10
|
+
if PumaPluginSupport.puma_plugin_enabled?
|
|
17
11
|
if Rails.env.production?
|
|
18
12
|
warn(
|
|
19
|
-
|
|
13
|
+
check_id,
|
|
20
14
|
'Solid Queue Puma plugin is enabled in production',
|
|
21
15
|
suggestion: 'Run Solid Queue in a dedicated job process for better isolation and memory management'
|
|
22
16
|
)
|
|
23
17
|
else
|
|
24
|
-
pass(
|
|
18
|
+
pass(check_id, 'Solid Queue Puma plugin detected (non-production environment)')
|
|
25
19
|
end
|
|
26
20
|
else
|
|
27
|
-
pass(
|
|
21
|
+
pass(check_id, 'Solid Queue is not co-located with Puma')
|
|
28
22
|
end
|
|
29
23
|
end
|
|
30
24
|
end
|
|
@@ -5,18 +5,91 @@ module SolidQueueGuard
|
|
|
5
5
|
module Config
|
|
6
6
|
class QueueSchemaCheck < Base
|
|
7
7
|
def call
|
|
8
|
-
|
|
8
|
+
evaluation = evaluate_schema
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
case evaluation[:status]
|
|
11
|
+
when :pass
|
|
12
|
+
pass(check_id, evaluation[:message], metadata: evaluation[:metadata])
|
|
13
|
+
when :warn
|
|
13
14
|
warn(
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
suggestion:
|
|
15
|
+
check_id,
|
|
16
|
+
evaluation[:message],
|
|
17
|
+
suggestion: evaluation[:suggestion],
|
|
18
|
+
metadata: evaluation[:metadata]
|
|
19
|
+
)
|
|
20
|
+
else
|
|
21
|
+
failure(
|
|
22
|
+
check_id,
|
|
23
|
+
evaluation[:message],
|
|
24
|
+
suggestion: evaluation[:suggestion],
|
|
25
|
+
metadata: evaluation[:metadata]
|
|
17
26
|
)
|
|
18
27
|
end
|
|
19
28
|
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def evaluate_schema
|
|
33
|
+
required = Schema::SolidQueueTables.required_tables
|
|
34
|
+
detection = Schema::SolidQueueTables.detected_tables(rails_root: rails_root)
|
|
35
|
+
missing = required - detection[:tables]
|
|
36
|
+
metadata = schema_metadata(detection, missing: missing)
|
|
37
|
+
|
|
38
|
+
return pass_evaluation(detection, required.size, metadata) if missing.empty?
|
|
39
|
+
return partial_evaluation(missing, metadata) if detection[:tables].any?
|
|
40
|
+
|
|
41
|
+
empty_evaluation(metadata)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def pass_evaluation(detection, table_count, metadata)
|
|
45
|
+
{
|
|
46
|
+
status: :pass,
|
|
47
|
+
message: success_message(detection, table_count),
|
|
48
|
+
metadata: metadata
|
|
49
|
+
}
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def partial_evaluation(missing, metadata)
|
|
53
|
+
{
|
|
54
|
+
status: :warn,
|
|
55
|
+
message: "Missing Solid Queue tables: #{missing.join(', ')}",
|
|
56
|
+
suggestion: 'Run bin/rails solid_queue:install or migrate the queue database',
|
|
57
|
+
metadata: metadata
|
|
58
|
+
}
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def empty_evaluation(metadata)
|
|
62
|
+
{
|
|
63
|
+
status: :fail,
|
|
64
|
+
message: [
|
|
65
|
+
'No Solid Queue schema tables found in',
|
|
66
|
+
'db/queue_schema.rb, db/schema.rb, db/structure.sql, or the queue database'
|
|
67
|
+
].join(' '),
|
|
68
|
+
suggestion: 'Run bin/rails solid_queue:install and migrate the queue database',
|
|
69
|
+
metadata: metadata
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def success_message(detection, table_count)
|
|
74
|
+
sources = source_descriptions(detection)
|
|
75
|
+
"Solid Queue schema complete (#{table_count}/#{table_count} tables) in #{sources.join(', ')}"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def source_descriptions(detection)
|
|
79
|
+
sources = detection[:file_tables].keys
|
|
80
|
+
sources << 'queue database' if detection[:database_tables].any?
|
|
81
|
+
sources.presence || ['schema files']
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def schema_metadata(detection, missing:)
|
|
85
|
+
{
|
|
86
|
+
required_tables: Schema::SolidQueueTables.required_tables,
|
|
87
|
+
detected_tables: detection[:tables],
|
|
88
|
+
missing_tables: missing,
|
|
89
|
+
schema_sources: detection[:file_tables],
|
|
90
|
+
database_tables: detection[:database_tables]
|
|
91
|
+
}
|
|
92
|
+
end
|
|
20
93
|
end
|
|
21
94
|
end
|
|
22
95
|
end
|
|
@@ -15,7 +15,8 @@ module SolidQueueGuard
|
|
|
15
15
|
Config::EnvFlagsCheck,
|
|
16
16
|
Config::ProcessHeartbeatConfigCheck,
|
|
17
17
|
Config::PumaColocatedCheck,
|
|
18
|
-
Config::TopologyRecommendationCheck
|
|
18
|
+
Config::TopologyRecommendationCheck,
|
|
19
|
+
Config::AsyncSupervisorConfigCheck
|
|
19
20
|
].freeze
|
|
20
21
|
|
|
21
22
|
RUNTIME_CHECKS = [
|
|
@@ -30,10 +31,15 @@ module SolidQueueGuard
|
|
|
30
31
|
Runtime::RecurringStaleCheck,
|
|
31
32
|
Runtime::PausedQueueLagCheck,
|
|
32
33
|
Runtime::PidfileCheck,
|
|
33
|
-
Runtime::FinishedJobsGrowthCheck
|
|
34
|
+
Runtime::FinishedJobsGrowthCheck,
|
|
35
|
+
Runtime::PumaPluginRuntimeCheck
|
|
34
36
|
].freeze
|
|
35
37
|
|
|
36
38
|
class << self
|
|
39
|
+
def check_id_for(check_class)
|
|
40
|
+
check_class.check_id
|
|
41
|
+
end
|
|
42
|
+
|
|
37
43
|
def for(scope)
|
|
38
44
|
case scope.to_sym
|
|
39
45
|
when :config then CONFIG_CHECKS
|
|
@@ -28,8 +28,11 @@ module SolidQueueGuard
|
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
def lag_threshold_for(queue_name)
|
|
31
|
-
thresholds = config.queue_lag_thresholds
|
|
32
|
-
thresholds[queue_name.to_sym] || thresholds[queue_name]
|
|
31
|
+
thresholds = config.check_setting(:queue_lag, :thresholds, config.queue_lag_thresholds)
|
|
32
|
+
per_queue = thresholds[queue_name.to_sym] || thresholds[queue_name]
|
|
33
|
+
return per_queue if per_queue
|
|
34
|
+
|
|
35
|
+
config.check_setting(:queue_lag, :threshold, thresholds[:default])
|
|
33
36
|
end
|
|
34
37
|
end
|
|
35
38
|
end
|
|
@@ -9,16 +9,18 @@ module SolidQueueGuard
|
|
|
9
9
|
dispatchers = SolidQueue::Process.where(kind: 'Dispatcher')
|
|
10
10
|
due_count = SolidQueue::ScheduledExecution.due.count
|
|
11
11
|
|
|
12
|
+
threshold = config.check_setting(:scheduled_backlog, :threshold, config.scheduled_backlog_threshold)
|
|
13
|
+
|
|
12
14
|
if dispatchers.none? && due_count.positive?
|
|
13
15
|
failure(
|
|
14
16
|
check_id,
|
|
15
17
|
"#{due_count} scheduled job(s) are due but no dispatcher is running",
|
|
16
18
|
suggestion: 'Start a dispatcher process or verify bin/jobs is running'
|
|
17
19
|
)
|
|
18
|
-
elsif due_count >
|
|
20
|
+
elsif due_count > threshold
|
|
19
21
|
warn(
|
|
20
22
|
check_id,
|
|
21
|
-
"#{due_count} scheduled executions are due (threshold: #{
|
|
23
|
+
"#{due_count} scheduled executions are due (threshold: #{threshold})",
|
|
22
24
|
suggestion: 'Verify the dispatcher is keeping up with scheduled work'
|
|
23
25
|
)
|
|
24
26
|
else
|
|
@@ -9,7 +9,7 @@ module SolidQueueGuard
|
|
|
9
9
|
def call
|
|
10
10
|
with_queue_database do
|
|
11
11
|
count = SolidQueue::FailedExecution.where(created_at: WINDOW.ago..).count
|
|
12
|
-
threshold = config.failed_jobs_threshold
|
|
12
|
+
threshold = config.check_setting(:failed_jobs, :threshold, config.failed_jobs_threshold)
|
|
13
13
|
|
|
14
14
|
if count > threshold
|
|
15
15
|
warn(
|