sidekiq-vigil 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.rubocop.yml +39 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +237 -0
  5. data/Rakefile +10 -0
  6. data/docs/redis_keys.md +18 -0
  7. data/docs/webhook_event.schema.json +41 -0
  8. data/docs/webhook_schema.md +64 -0
  9. data/exe/vigil +6 -0
  10. data/lib/generators/sidekiq_vigil/install_generator.rb +32 -0
  11. data/lib/generators/sidekiq_vigil/templates/sidekiq_vigil.rb +49 -0
  12. data/lib/sidekiq/vigil/version.rb +7 -0
  13. data/lib/sidekiq/vigil.rb +7 -0
  14. data/lib/sidekiq_vigil/alert/cron.rb +106 -0
  15. data/lib/sidekiq_vigil/alert/event.rb +110 -0
  16. data/lib/sidekiq_vigil/alert/grouping.rb +15 -0
  17. data/lib/sidekiq_vigil/alert/manager.rb +227 -0
  18. data/lib/sidekiq_vigil/alert/mute.rb +69 -0
  19. data/lib/sidekiq_vigil/alert/state.rb +51 -0
  20. data/lib/sidekiq_vigil/check/base.rb +52 -0
  21. data/lib/sidekiq_vigil/check/failure_rate.rb +45 -0
  22. data/lib/sidekiq_vigil/check/memory.rb +41 -0
  23. data/lib/sidekiq_vigil/check/process_alive.rb +64 -0
  24. data/lib/sidekiq_vigil/check/queue_latency.rb +21 -0
  25. data/lib/sidekiq_vigil/check/queue_size.rb +20 -0
  26. data/lib/sidekiq_vigil/check/redis_health.rb +55 -0
  27. data/lib/sidekiq_vigil/check/registry.rb +23 -0
  28. data/lib/sidekiq_vigil/check/scheduled_backlog.rb +21 -0
  29. data/lib/sidekiq_vigil/check/set_size.rb +76 -0
  30. data/lib/sidekiq_vigil/check/stuck_jobs.rb +44 -0
  31. data/lib/sidekiq_vigil/check/throughput_anomaly.rb +69 -0
  32. data/lib/sidekiq_vigil/check/utilization.rb +61 -0
  33. data/lib/sidekiq_vigil/checker.rb +149 -0
  34. data/lib/sidekiq_vigil/cli.rb +205 -0
  35. data/lib/sidekiq_vigil/collector.rb +74 -0
  36. data/lib/sidekiq_vigil/config.rb +267 -0
  37. data/lib/sidekiq_vigil/health_app.rb +64 -0
  38. data/lib/sidekiq_vigil/leader_election.rb +54 -0
  39. data/lib/sidekiq_vigil/middleware/server.rb +37 -0
  40. data/lib/sidekiq_vigil/notifier/base.rb +22 -0
  41. data/lib/sidekiq_vigil/notifier/http_transport.rb +32 -0
  42. data/lib/sidekiq_vigil/notifier/log.rb +14 -0
  43. data/lib/sidekiq_vigil/notifier/manager.rb +45 -0
  44. data/lib/sidekiq_vigil/notifier/slack.rb +192 -0
  45. data/lib/sidekiq_vigil/notifier/webhook.rb +25 -0
  46. data/lib/sidekiq_vigil/reporter.rb +103 -0
  47. data/lib/sidekiq_vigil/result.rb +43 -0
  48. data/lib/sidekiq_vigil/runtime.rb +44 -0
  49. data/lib/sidekiq_vigil/sidekiq_api.rb +40 -0
  50. data/lib/sidekiq_vigil/storage.rb +167 -0
  51. data/lib/sidekiq_vigil/timezone.rb +39 -0
  52. data/lib/sidekiq_vigil/version.rb +5 -0
  53. data/lib/sidekiq_vigil.rb +128 -0
  54. metadata +111 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 9ba1173ca461f02e3504da159ae2d276024e06abe21498ed68cd366f81451474
4
+ data.tar.gz: be2bd213fe4c4687a06acfbb706f3b7847df9e913d0028f14ff25da4cce24bed
5
+ SHA512:
6
+ metadata.gz: ebec4298778df2aeee2a848b20cbec83a8224de9946849d0a04f436e32466f8bccbb8c663a3b58b2146f41e00d13a446d8135cb26e202679c2e846807dc6bef9
7
+ data.tar.gz: bff257da92bb81f5036d7a3ea6d2a4d64a7edb4dcef71d1d135a5a0d58a3fe90636e20b6b8bb08cc1c50a130c07bd61946a45e60dd20e3a546a6aa087908c1bd
data/.rubocop.yml ADDED
@@ -0,0 +1,39 @@
1
+ AllCops:
2
+ NewCops: enable
3
+ TargetRubyVersion: 3.2
4
+ SuggestExtensions: false
5
+ UseCache: false
6
+ Exclude:
7
+ - "vendor/**/*"
8
+ - "gemfiles/**/*"
9
+
10
+ Layout/LineLength:
11
+ Max: 140
12
+
13
+ Metrics/BlockLength:
14
+ Exclude:
15
+ - "spec/**/*"
16
+ - "*.gemspec"
17
+ - "lib/generators/**/templates/*.rb"
18
+
19
+ Metrics/ClassLength:
20
+ Max: 180
21
+
22
+ Metrics/AbcSize:
23
+ Max: 30
24
+
25
+ Metrics/MethodLength:
26
+ Max: 35
27
+
28
+ Metrics/ParameterLists:
29
+ Max: 8
30
+
31
+ Naming/PredicateMethod:
32
+ AllowedMethods:
33
+ - notify
34
+
35
+ Style/StringLiterals:
36
+ EnforcedStyle: double_quotes
37
+
38
+ Style/Documentation:
39
+ Enabled: false
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2026 Yudai Takada
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,237 @@
1
+ # Sidekiq Vigil
2
+
3
+ Sidekiq Vigil is battery-included monitoring and alerting for Sidekiq. It runs against the Redis you already use, evaluates twelve health checks, keeps an alert lifecycle across restarts, and sends firing, ongoing, and recovery notifications.
4
+
5
+ [![CI](https://github.com/ydah/sidekiq-vigil/actions/workflows/main.yml/badge.svg)](https://github.com/ydah/sidekiq-vigil/actions/workflows/main.yml)
6
+
7
+ ## Requirements
8
+
9
+ - Ruby 3.2 or newer
10
+ - Sidekiq 7.x or 8.x
11
+ - Redis 7 or a Sidekiq-compatible Redis implementation
12
+
13
+ Sidekiq is the only runtime dependency.
14
+
15
+ ## Five-minute quick start
16
+
17
+ Add the gem from Git until the first RubyGems release:
18
+
19
+ ```ruby
20
+ gem "sidekiq-vigil", github: "ydah/sidekiq-vigil"
21
+ ```
22
+
23
+ Generate an initializer in Rails:
24
+
25
+ ```sh
26
+ bundle install
27
+ bundle exec rails generate sidekiq_vigil:install
28
+ ```
29
+
30
+ Or create a pure Ruby initializer:
31
+
32
+ ```ruby
33
+ require "sidekiq_vigil"
34
+
35
+ SidekiqVigil.configure do |config|
36
+ config.key_prefix = "myapp"
37
+ config.redis = { url: ENV["REDIS_URL"] }
38
+
39
+ config.notifier :slack,
40
+ webhook_url: ENV["SLACK_WEBHOOK_DEFAULT"],
41
+ routes: { critical: ENV["SLACK_WEBHOOK_INCIDENTS"] }.compact,
42
+ mention: { critical: "<!here>" },
43
+ web_ui_url: "https://example.com/sidekiq"
44
+ config.notifier :log
45
+
46
+ config.check :queue_latency, warn: 60, critical: 300
47
+ config.check :queue_size, warn: 1_000, critical: 10_000
48
+ end
49
+ ```
50
+
51
+ Start Sidekiq normally. The server middleware, Reporter, and leader-elected Checker are installed automatically. Outside `production`, external notification is disabled unless `config.enabled = true` is assigned explicitly; Log remains available.
52
+
53
+ Test every configured notifier before relying on it:
54
+
55
+ ```sh
56
+ bundle exec vigil --config config/sidekiq_vigil.rb test-notify
57
+ ```
58
+
59
+ ## Recommended production topology
60
+
61
+ Use Embedded mode for collection and checks, expose `SidekiqVigil::HealthApp`, and have an external uptime monitor call `/healthz`. Embedded monitoring cannot run when every Sidekiq process is dead; the endpoint detects that condition because an expired snapshot returns 503.
62
+
63
+ ```ruby
64
+ # config.ru
65
+ require "sidekiq_vigil"
66
+
67
+ storage = SidekiqVigil.build_storage
68
+ health = SidekiqVigil::HealthApp.new(
69
+ storage: storage,
70
+ interval: SidekiqVigil.config.interval
71
+ )
72
+
73
+ # Authentication is intentionally supplied by the application.
74
+ use Rack::Auth::Basic do |username, password|
75
+ Rack::Utils.secure_compare(username, ENV.fetch("VIGIL_HEALTH_USER")) &&
76
+ Rack::Utils.secure_compare(password, ENV.fetch("VIGIL_HEALTH_PASSWORD"))
77
+ end
78
+ run health
79
+ ```
80
+
81
+ `GET /healthz` returns 200 only when every result is OK and the snapshot is at most `interval × 2` old. `GET /status.json` includes check results, freshness, and age. Both return 503 when unhealthy or unavailable.
82
+
83
+ ## Execution modes
84
+
85
+ | Mode | Command/integration | Collection | Detects total Sidekiq outage |
86
+ |---|---|---:|---:|
87
+ | Embedded (default) | Require the gem in Sidekiq | Reporter + leader Checker | Through stale external `/healthz` probe |
88
+ | Standalone | `vigil --config config/vigil.rb` | Checker only | Yes |
89
+ | Oneshot | `vigil --config config/vigil.rb check --once` | One Checker cycle | Yes, when scheduled externally |
90
+
91
+ Standalone configuration is plain Ruby and must not depend on Rails constants.
92
+
93
+ ```sh
94
+ bundle exec vigil --config config/vigil.rb status
95
+ bundle exec vigil --config config/vigil.rb mute 30m --reason "database maintenance"
96
+ bundle exec vigil --config config/vigil.rb unmute
97
+ ```
98
+
99
+ `TERM` and `INT` stop Standalone mode gracefully and release its leader lock.
100
+
101
+ ## Check catalog
102
+
103
+ | Check | Default behavior | Important options |
104
+ |---|---|---|
105
+ | `queue_latency` | Warn 60s, critical 300s | `queues`, `per_queue` |
106
+ | `queue_size` | Warn 1,000, critical 10,000 | `queues`, `per_queue` |
107
+ | `retry_set` | Warn 100, critical 1,000 | `growth_only` |
108
+ | `dead_set` | Warn 1, critical 50 | `growth_only` |
109
+ | `process_alive` | Require at least one active process | `min_processes`, `quiet_threshold` |
110
+ | `utilization` | Warn 85%, critical 95% after 300s | `sustained` |
111
+ | `failure_rate` | Warn 5%, critical 20%, minimum 20 attempts | `window`, `min_samples` |
112
+ | `stuck_jobs` | Critical after 30 minutes | `threshold` |
113
+ | `memory` | Warn 1,500 MiB, critical 2,500 MiB | `warn_mb`, `critical_mb` |
114
+ | `redis_health` | PING latency and maxmemory usage | `latency_ms`, `memory_pct` |
115
+ | `scheduled_backlog` | Warn on jobs overdue by five minutes | `overdue` |
116
+ | `throughput_anomaly` | Disabled by default | `drop_pct`, `baseline_days`, `min_samples` |
117
+
118
+ All checks use Sidekiq's public `sidekiq/api` objects. A check exception becomes an `error` result; it does not terminate Sidekiq or the Checker loop. Custom checks subclass `SidekiqVigil::Check::Base` and return one `SidekiqVigil::Result` or an array.
119
+
120
+ Failure rate counts execution attempts, so a job that fails and retries can contribute more than one failure. Stuck-job duration assumes synchronized host clocks (NTP).
121
+
122
+ ## Full configuration reference
123
+
124
+ ```ruby
125
+ SidekiqVigil.configure do |config|
126
+ config.enabled = true
127
+ config.interval = 30
128
+ config.flush_interval = 10
129
+ config.key_prefix = "myapp"
130
+ config.timezone = "Asia/Tokyo"
131
+ config.redis = {
132
+ url: ENV["REDIS_URL"],
133
+ pool_size: 5,
134
+ pool_timeout: 1
135
+ }
136
+
137
+ config.notifier :slack,
138
+ webhook_url: ENV["SLACK_WEBHOOK_DEFAULT"],
139
+ routes: { critical: ENV["SLACK_WEBHOOK_INCIDENTS"] },
140
+ mention: { critical: "<!subteam^S012345>" },
141
+ web_ui_url: "https://example.com/sidekiq"
142
+ config.notifier :webhook, url: "https://events.example.com/sidekiq"
143
+ config.notifier :log
144
+
145
+ config.alerting do |alerting|
146
+ alerting.pending_cycles = 2
147
+ alerting.cooldown = 600
148
+ alerting.resolve_notice = true
149
+ alerting.flap_window = 120
150
+ alerting.flap_threshold = 4
151
+ alerting.escalate_after = 3
152
+ alerting.group_threshold = 5
153
+ alerting.group_top_n = 5
154
+ alerting.mutes = [
155
+ { cron: "0 3 * * 0", duration: 3_600, reason: "weekly maintenance" }
156
+ ]
157
+ end
158
+ end
159
+ ```
160
+
161
+ Slack Incoming Webhooks have a fixed channel. Severity routing therefore selects different URLs; Sidekiq Vigil does not attempt to override `channel` in a payload. Mentions must use `<@U…>`, `<!subteam^S…>`, or `<!here>` syntax. Plain text such as `@oncall` fails configuration validation. Secret URLs are filtered from notifier inspection and error logging.
162
+
163
+ The generic webhook contract and examples are documented in [docs/webhook_schema.md](docs/webhook_schema.md).
164
+
165
+ The five Slack Block Kit shapes are contract-tested against JSON previews for [critical](spec/fixtures/slack/critical.json), [warn](spec/fixtures/slack/warn.json), [resolved](spec/fixtures/slack/resolved.json), [still-firing](spec/fixtures/slack/still_firing.json), and [digest](spec/fixtures/slack/digest.json). A real Slack screenshot must come from the manual pre-release check; generated imagery is not used as release evidence.
166
+
167
+ ## Alert lifecycle
168
+
169
+ Each `check_name + target` moves through `OK → PENDING → FIRING → RESOLVED → OK`. State survives leader changes and process restarts.
170
+
171
+ - `pending_cycles` removes single-cycle noise.
172
+ - `cooldown` controls ongoing notifications.
173
+ - `escalate_after` upgrades sustained warnings to critical.
174
+ - `flap_threshold` transitions inside `flap_window` emit one flapping event and hold refiring until the window closes.
175
+ - history retains the latest 30 values for 24 hours.
176
+ - targets that disappear are pruned every cycle.
177
+ - more than `group_threshold` events in one cycle become one digest with severity counts and `group_top_n` details.
178
+ - mute windows continue state transitions but suppress delivery; unmute reports current firing state once.
179
+
180
+ Mute cron expressions use the standard five fields and support lists (`1,3`), ranges (`1-5`), and steps (`*/10`). When both day-of-month and weekday are restricted, standard cron OR semantics apply.
181
+
182
+ All Redis keys start with `vigil:<key_prefix>:`. The Storage API rejects non-positive or missing TTLs except for the explicitly managed `alerts` hash. The complete, contract-tested key list is in [docs/redis_keys.md](docs/redis_keys.md).
183
+
184
+ ## Existing tools
185
+
186
+ | Tool | Primary purpose | Built-in lifecycle/notifications | Additional stack |
187
+ |---|---|---:|---:|
188
+ | yabeda-sidekiq / Prometheus exporters | Metrics export | No | Prometheus/Grafana/Alertmanager |
189
+ | Sidekiq Alive | Kubernetes liveness | No | External probe |
190
+ | sidekiq-job_alert | Queue/dead Slack alerts | Limited | No |
191
+ | Sidekiq Web metrics | Execution exploration | No | No |
192
+ | Sidekiq Vigil | Checks, lifecycle, mute/group/recovery | Yes | No |
193
+
194
+ ## Known limitations
195
+
196
+ - During complete Redis failure, each process loses leader coordination and direct `redis_health` notifications can be duplicated. Recovery may also cause another notification. Use an external `/healthz` probe as the primary total-outage signal.
197
+ - Throughput anomaly baseline uses the previous seven same-time windows and does not model weekday/weekend seasonality. It is opt-in and has a minimum-sample guard.
198
+ - Stuck-job detection compares timestamps from different hosts and requires synchronized clocks.
199
+ - RSS collection supports Linux `/proc` and macOS `ps`; the memory check is skipped on Windows.
200
+ - Sidekiq Pro and Enterprise use only the same public APIs, but the combinations are not yet separately certified.
201
+ - Slack Bot Token mode, thread tracking, weekday baselines, and Prometheus output are planned after v1.
202
+
203
+ ## Development
204
+
205
+ ```sh
206
+ bundle install
207
+ redis-server --port 16379 --save "" --appendonly no
208
+ VIGIL_REDIS_URL=redis://127.0.0.1:16379/15 bundle exec rake
209
+ bundle exec appraisal sidekiq_7 rspec
210
+ bundle exec appraisal sidekiq_8 rspec
211
+ bundle exec ruby benchmark/middleware_overhead.rb
212
+ bundle exec rake build
213
+ ```
214
+
215
+ The benchmark compares middleware-on and middleware-off in the same process, writes a JSON artifact, and only warns when the relative run has unusually high overhead.
216
+
217
+ ## FAQ
218
+
219
+ **Does the middleware call Redis for every job?**
220
+
221
+ No. It updates a mutex-protected in-process counter. Reporter flushes aggregates in a Redis pipeline.
222
+
223
+ **Can I run Embedded and Standalone together?**
224
+
225
+ Yes. The Redis leader lock prevents duplicate normal checks. A configuration digest warning appears when process configurations differ.
226
+
227
+ **Why did Slack not receive a development notification?**
228
+
229
+ External notifiers are safe by default outside production. Explicitly assign `config.enabled = true`, then run `vigil test-notify`.
230
+
231
+ **Can I keep using yabeda or an APM?**
232
+
233
+ Yes. Sidekiq Vigil does not replace or modify Sidekiq's metrics subsystem.
234
+
235
+ ## License
236
+
237
+ MIT. See [LICENSE.txt](LICENSE.txt).
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler/gem_tasks"
4
+ require "rspec/core/rake_task"
5
+ require "rubocop/rake_task"
6
+
7
+ RSpec::Core::RakeTask.new(:spec)
8
+ RuboCop::RakeTask.new(:rubocop)
9
+
10
+ task default: %i[spec rubocop]
@@ -0,0 +1,18 @@
1
+ # Redis key contract
2
+
3
+ Every key is namespaced below `vigil:<key_prefix>:`. Storage writes require a positive TTL; the `alerts` hash is the only persistent exception and is explicitly pruned by `Alert::Manager`.
4
+
5
+ The first column of this table is contract-tested against `SidekiqVigil::Storage::KEY_CATALOG`.
6
+
7
+ | Key suffix | Type | Content | TTL | Owner |
8
+ |---|---|---|---|---|
9
+ | `stats:{yyyymmddHHMM}` | hash | Minute counters for processed/failed jobs and worker classes | 8 days | Reporter |
10
+ | `exec:{queue}` | hash | Execution count/sum/max by queue | 1 hour | Reporter |
11
+ | `alerts` | hash | Serialized alert lifecycle state | Explicitly managed; stale targets are pruned | Alert::Manager |
12
+ | `history:{alert_id}` | list | Latest 30 observed values for a single alert | 24 hours | Alert::Manager |
13
+ | `snapshot` | string | Timestamped check results and alert states | `interval × 4` | Checker |
14
+ | `leader` | string | Leader-election token | `interval × 3` | LeaderElection |
15
+ | `mem:{process_id}` | string | Reporter RSS in KiB | `flush_interval × 3` | Reporter |
16
+ | `config_digest` | hash | Configuration digest by process | 1 hour | Reporter |
17
+ | `mute` | string | Manual mute reason and expiration | Requested duration | Alert::Mute |
18
+ | `check_state:{check-specific-suffix}` | string | Growth, quiet-process, or sustained-condition state | Check-specific positive TTL | Checks |
@@ -0,0 +1,41 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://github.com/ydah/sidekiq-vigil/blob/main/docs/webhook_event.schema.json",
4
+ "title": "Sidekiq Vigil webhook event",
5
+ "type": "object",
6
+ "required": ["schema_version", "event", "alert_id", "timestamp"],
7
+ "properties": {
8
+ "schema_version": { "const": "1.0" },
9
+ "event": {
10
+ "enum": ["firing", "still_firing", "resolved", "escalated", "flapping", "unmuted", "digest"]
11
+ },
12
+ "alert_id": { "type": "string", "minLength": 1 },
13
+ "timestamp": { "type": "string", "format": "date-time" },
14
+ "alert": {
15
+ "type": "object",
16
+ "required": ["check_name", "target", "severity", "metadata"],
17
+ "properties": {
18
+ "check_name": { "type": "string" },
19
+ "target": { "type": "string" },
20
+ "severity": { "enum": ["ok", "warn", "critical", "error"] },
21
+ "value": {},
22
+ "threshold": {},
23
+ "message": { "type": ["string", "null"] },
24
+ "metadata": { "type": "object" }
25
+ },
26
+ "additionalProperties": false
27
+ },
28
+ "state": { "type": "object" },
29
+ "history": { "type": "array" },
30
+ "counts": { "type": "object" },
31
+ "alerts": { "type": "array" }
32
+ },
33
+ "allOf": [
34
+ {
35
+ "if": { "properties": { "event": { "const": "digest" } } },
36
+ "then": { "required": ["counts", "alerts"] },
37
+ "else": { "required": ["alert", "state", "history"] }
38
+ }
39
+ ],
40
+ "additionalProperties": false
41
+ }
@@ -0,0 +1,64 @@
1
+ # Webhook payload schema
2
+
3
+ The generic webhook notifier sends one JSON object per alert event. The normative JSON Schema is [`webhook_event.schema.json`](webhook_event.schema.json).
4
+
5
+ ## Compatibility
6
+
7
+ - `schema_version` is currently `1.0`.
8
+ - New optional properties may be added in a minor release.
9
+ - Removing a property, changing its type, or changing event semantics requires a schema major-version change.
10
+ - Receivers should ignore unknown optional properties.
11
+
12
+ ## Regular event
13
+
14
+ ```json
15
+ {
16
+ "schema_version": "1.0",
17
+ "event": "firing",
18
+ "alert_id": "queue_size:critical",
19
+ "timestamp": "2026-07-28T12:00:00Z",
20
+ "alert": {
21
+ "check_name": "queue_size",
22
+ "target": "critical",
23
+ "severity": "critical",
24
+ "value": 250,
25
+ "threshold": 200,
26
+ "message": "250 jobs waiting",
27
+ "metadata": {}
28
+ },
29
+ "state": {
30
+ "status": "firing",
31
+ "cycles": 2,
32
+ "first_seen_at": 1785239970.0,
33
+ "last_notified_at": 1785240000.0,
34
+ "last_transition_at": 1785240000.0,
35
+ "severity": "critical",
36
+ "value": 250,
37
+ "threshold": 200,
38
+ "message": "250 jobs waiting",
39
+ "suppressed": false,
40
+ "flap_notified": false,
41
+ "flapping_until": null,
42
+ "transition_timestamps": [1785240000.0]
43
+ },
44
+ "history": [
45
+ { "timestamp": 1785239970.0, "value": 190, "severity": "warn" },
46
+ { "timestamp": 1785240000.0, "value": 250, "severity": "critical" }
47
+ ]
48
+ }
49
+ ```
50
+
51
+ ## Digest event
52
+
53
+ A digest replaces individual notifications when the number of events in one checker cycle is greater than `group_threshold`. It contains severity counts and the highest-severity alerts.
54
+
55
+ ```json
56
+ {
57
+ "schema_version": "1.0",
58
+ "event": "digest",
59
+ "alert_id": "digest",
60
+ "timestamp": "2026-07-28T12:00:00Z",
61
+ "counts": { "critical": 3, "warn": 4 },
62
+ "alerts": []
63
+ }
64
+ ```
data/exe/vigil ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "sidekiq_vigil"
5
+
6
+ exit SidekiqVigil::CLI.new.run(ARGV)
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+
5
+ begin
6
+ require "rails/generators"
7
+
8
+ module SidekiqVigil
9
+ module Generators
10
+ class InstallGenerator < Rails::Generators::Base
11
+ source_root File.expand_path("templates", __dir__)
12
+
13
+ def copy_initializer
14
+ template "sidekiq_vigil.rb", "config/initializers/sidekiq_vigil.rb"
15
+ end
16
+ end
17
+ end
18
+ end
19
+ rescue LoadError
20
+ module SidekiqVigil
21
+ module Generators
22
+ class InstallGenerator
23
+ TEMPLATE = File.expand_path("templates/sidekiq_vigil.rb", __dir__)
24
+
25
+ def self.install(destination)
26
+ FileUtils.mkdir_p(File.dirname(destination))
27
+ FileUtils.cp(TEMPLATE, destination)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sidekiq_vigil"
4
+
5
+ SidekiqVigil.configure do |config|
6
+ # Monitoring is enabled by default. External notification is enabled by
7
+ # default only in production; set this explicitly to opt in elsewhere.
8
+ config.enabled = ENV.fetch("VIGIL_ENABLED") == "true" if ENV.key?("VIGIL_ENABLED")
9
+ config.interval = Integer(ENV.fetch("VIGIL_INTERVAL", "30"))
10
+ config.flush_interval = Integer(ENV.fetch("VIGIL_FLUSH_INTERVAL", "10"))
11
+ config.key_prefix = ENV.fetch("VIGIL_KEY_PREFIX", "myapp")
12
+ config.timezone = ENV.fetch("VIGIL_TIMEZONE", "UTC")
13
+ redis_url = ENV.fetch("REDIS_URL", nil)
14
+ config.redis = { url: redis_url } if redis_url
15
+
16
+ config.check :queue_latency, warn: 60, critical: 300
17
+ config.check :queue_size, warn: 1_000, critical: 10_000
18
+ config.check :retry_set, warn: 100, critical: 1_000
19
+ config.check :dead_set, warn: 1, critical: 50, growth_only: true
20
+ config.check :process_alive, min_processes: 1
21
+ config.check :utilization, warn: 0.85, critical: 0.95, sustained: 300
22
+ config.check :failure_rate, warn: 0.05, critical: 0.20, window: 300, min_samples: 20
23
+ config.check :stuck_jobs, threshold: 1_800
24
+ config.check :memory, warn_mb: 1_500, critical_mb: 2_500
25
+ config.check :redis_health, latency_ms: 100, memory_pct: 0.9
26
+ config.check :scheduled_backlog, overdue: 300
27
+ # Opt in after enough history has accumulated:
28
+ # config.check :throughput_anomaly, drop_pct: 0.5, baseline_days: 7
29
+
30
+ slack_webhook = ENV.fetch("SLACK_WEBHOOK_DEFAULT", nil)
31
+ if slack_webhook
32
+ config.notifier :slack,
33
+ webhook_url: slack_webhook,
34
+ routes: { critical: ENV.fetch("SLACK_WEBHOOK_INCIDENTS", nil) }.compact,
35
+ mention: { critical: ENV.fetch("SLACK_CRITICAL_MENTION", nil) }.compact,
36
+ web_ui_url: ENV.fetch("SIDEKIQ_WEB_UI_URL", nil)
37
+ end
38
+ config.notifier :log
39
+
40
+ config.alerting do |alerting|
41
+ alerting.pending_cycles = 2
42
+ alerting.cooldown = 600
43
+ alerting.resolve_notice = true
44
+ alerting.flap_window = 120
45
+ alerting.flap_threshold = 4
46
+ alerting.group_threshold = 5
47
+ alerting.group_top_n = 5
48
+ end
49
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../../sidekiq_vigil/version"
4
+
5
+ module Sidekiq
6
+ Vigil = SidekiqVigil unless const_defined?(:Vigil)
7
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../sidekiq_vigil"
4
+
5
+ module Sidekiq
6
+ Vigil = SidekiqVigil unless const_defined?(:Vigil)
7
+ end
@@ -0,0 +1,106 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqVigil
4
+ module Alert
5
+ class Cron
6
+ Field = Data.define(:values, :wildcard)
7
+
8
+ FIELD_RANGES = [
9
+ (0..59),
10
+ (0..23),
11
+ (1..31),
12
+ (1..12),
13
+ (0..7)
14
+ ].freeze
15
+
16
+ def initialize(expression)
17
+ fields = expression.to_s.split
18
+ raise ConfigError, "mute cron must contain five fields" unless fields.length == FIELD_RANGES.length
19
+
20
+ @fields = fields.zip(FIELD_RANGES).map { |field, range| parse_field(field, range) }
21
+ end
22
+
23
+ def match?(time)
24
+ minute, hour, day, month, weekday = fields
25
+ fixed_fields = [[minute, time.min], [hour, time.hour], [month, time.month]]
26
+ return false unless fixed_fields.all? { |field, value| field.values.include?(value) }
27
+
28
+ day_matches = day.values.include?(time.day)
29
+ weekday_matches = weekday.values.include?(time.wday)
30
+ return day_matches || weekday_matches unless day.wildcard || weekday.wildcard
31
+
32
+ day_matches && weekday_matches
33
+ end
34
+
35
+ private
36
+
37
+ attr_reader :fields
38
+
39
+ def parse_field(source, range)
40
+ wildcard = source.start_with?("*")
41
+ values = source.split(",").flat_map { |part| expand_part(part, range) }.map do |value|
42
+ normalize_weekday(value, range)
43
+ end
44
+ raise ConfigError, "mute cron field #{source.inspect} selects no values" if values.empty?
45
+
46
+ Field.new(values: values.uniq.freeze, wildcard:)
47
+ rescue ArgumentError
48
+ raise ConfigError, "invalid mute cron field #{source.inspect}"
49
+ end
50
+
51
+ def expand_part(part, range)
52
+ base, raw_step = split_part(part)
53
+ step = parse_step(raw_step)
54
+ values = base_values(base, range, stepped: !raw_step.nil?)
55
+ values.each_with_index.filter_map { |value, index| value if (index % step).zero? }
56
+ end
57
+
58
+ def split_part(part)
59
+ base, raw_step, extra = part.split("/", 3)
60
+ raise ArgumentError if base.nil? || base.empty? || extra
61
+
62
+ [base, raw_step]
63
+ end
64
+
65
+ def parse_step(raw_step)
66
+ return 1 unless raw_step
67
+
68
+ step = Integer(raw_step, 10)
69
+ raise ArgumentError unless step.positive?
70
+
71
+ step
72
+ end
73
+
74
+ def base_values(base, range, stepped:)
75
+ return range.to_a if base == "*"
76
+
77
+ if base.include?("-")
78
+ first, last, extra = base.split("-", 3)
79
+ raise ArgumentError if extra
80
+
81
+ first = bounded_integer(first, range)
82
+ last = bounded_integer(last, range)
83
+ raise ArgumentError if first > last
84
+
85
+ return (first..last).to_a
86
+ end
87
+
88
+ first = bounded_integer(base, range)
89
+ return (first..range.end).to_a if stepped
90
+
91
+ [first]
92
+ end
93
+
94
+ def bounded_integer(value, range)
95
+ integer = Integer(value, 10)
96
+ raise ArgumentError unless range.cover?(integer)
97
+
98
+ integer
99
+ end
100
+
101
+ def normalize_weekday(value, range)
102
+ range.end == 7 && value == 7 ? 0 : value
103
+ end
104
+ end
105
+ end
106
+ end