sidekiq-vigil 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rubocop.yml +39 -0
- data/LICENSE.txt +21 -0
- data/README.md +237 -0
- data/Rakefile +10 -0
- data/docs/redis_keys.md +18 -0
- data/docs/webhook_event.schema.json +41 -0
- data/docs/webhook_schema.md +64 -0
- data/exe/vigil +6 -0
- data/lib/generators/sidekiq_vigil/install_generator.rb +32 -0
- data/lib/generators/sidekiq_vigil/templates/sidekiq_vigil.rb +49 -0
- data/lib/sidekiq/vigil/version.rb +7 -0
- data/lib/sidekiq/vigil.rb +7 -0
- data/lib/sidekiq_vigil/alert/cron.rb +106 -0
- data/lib/sidekiq_vigil/alert/event.rb +110 -0
- data/lib/sidekiq_vigil/alert/grouping.rb +15 -0
- data/lib/sidekiq_vigil/alert/manager.rb +227 -0
- data/lib/sidekiq_vigil/alert/mute.rb +69 -0
- data/lib/sidekiq_vigil/alert/state.rb +51 -0
- data/lib/sidekiq_vigil/check/base.rb +52 -0
- data/lib/sidekiq_vigil/check/failure_rate.rb +45 -0
- data/lib/sidekiq_vigil/check/memory.rb +41 -0
- data/lib/sidekiq_vigil/check/process_alive.rb +64 -0
- data/lib/sidekiq_vigil/check/queue_latency.rb +21 -0
- data/lib/sidekiq_vigil/check/queue_size.rb +20 -0
- data/lib/sidekiq_vigil/check/redis_health.rb +55 -0
- data/lib/sidekiq_vigil/check/registry.rb +23 -0
- data/lib/sidekiq_vigil/check/scheduled_backlog.rb +21 -0
- data/lib/sidekiq_vigil/check/set_size.rb +76 -0
- data/lib/sidekiq_vigil/check/stuck_jobs.rb +44 -0
- data/lib/sidekiq_vigil/check/throughput_anomaly.rb +69 -0
- data/lib/sidekiq_vigil/check/utilization.rb +61 -0
- data/lib/sidekiq_vigil/checker.rb +149 -0
- data/lib/sidekiq_vigil/cli.rb +205 -0
- data/lib/sidekiq_vigil/collector.rb +74 -0
- data/lib/sidekiq_vigil/config.rb +267 -0
- data/lib/sidekiq_vigil/health_app.rb +64 -0
- data/lib/sidekiq_vigil/leader_election.rb +54 -0
- data/lib/sidekiq_vigil/middleware/server.rb +37 -0
- data/lib/sidekiq_vigil/notifier/base.rb +22 -0
- data/lib/sidekiq_vigil/notifier/http_transport.rb +32 -0
- data/lib/sidekiq_vigil/notifier/log.rb +14 -0
- data/lib/sidekiq_vigil/notifier/manager.rb +45 -0
- data/lib/sidekiq_vigil/notifier/slack.rb +192 -0
- data/lib/sidekiq_vigil/notifier/webhook.rb +25 -0
- data/lib/sidekiq_vigil/reporter.rb +103 -0
- data/lib/sidekiq_vigil/result.rb +43 -0
- data/lib/sidekiq_vigil/runtime.rb +44 -0
- data/lib/sidekiq_vigil/sidekiq_api.rb +40 -0
- data/lib/sidekiq_vigil/storage.rb +167 -0
- data/lib/sidekiq_vigil/timezone.rb +39 -0
- data/lib/sidekiq_vigil/version.rb +5 -0
- data/lib/sidekiq_vigil.rb +128 -0
- metadata +111 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 9ba1173ca461f02e3504da159ae2d276024e06abe21498ed68cd366f81451474
|
|
4
|
+
data.tar.gz: be2bd213fe4c4687a06acfbb706f3b7847df9e913d0028f14ff25da4cce24bed
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: ebec4298778df2aeee2a848b20cbec83a8224de9946849d0a04f436e32466f8bccbb8c663a3b58b2146f41e00d13a446d8135cb26e202679c2e846807dc6bef9
|
|
7
|
+
data.tar.gz: bff257da92bb81f5036d7a3ea6d2a4d64a7edb4dcef71d1d135a5a0d58a3fe90636e20b6b8bb08cc1c50a130c07bd61946a45e60dd20e3a546a6aa087908c1bd
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
AllCops:
|
|
2
|
+
NewCops: enable
|
|
3
|
+
TargetRubyVersion: 3.2
|
|
4
|
+
SuggestExtensions: false
|
|
5
|
+
UseCache: false
|
|
6
|
+
Exclude:
|
|
7
|
+
- "vendor/**/*"
|
|
8
|
+
- "gemfiles/**/*"
|
|
9
|
+
|
|
10
|
+
Layout/LineLength:
|
|
11
|
+
Max: 140
|
|
12
|
+
|
|
13
|
+
Metrics/BlockLength:
|
|
14
|
+
Exclude:
|
|
15
|
+
- "spec/**/*"
|
|
16
|
+
- "*.gemspec"
|
|
17
|
+
- "lib/generators/**/templates/*.rb"
|
|
18
|
+
|
|
19
|
+
Metrics/ClassLength:
|
|
20
|
+
Max: 180
|
|
21
|
+
|
|
22
|
+
Metrics/AbcSize:
|
|
23
|
+
Max: 30
|
|
24
|
+
|
|
25
|
+
Metrics/MethodLength:
|
|
26
|
+
Max: 35
|
|
27
|
+
|
|
28
|
+
Metrics/ParameterLists:
|
|
29
|
+
Max: 8
|
|
30
|
+
|
|
31
|
+
Naming/PredicateMethod:
|
|
32
|
+
AllowedMethods:
|
|
33
|
+
- notify
|
|
34
|
+
|
|
35
|
+
Style/StringLiterals:
|
|
36
|
+
EnforcedStyle: double_quotes
|
|
37
|
+
|
|
38
|
+
Style/Documentation:
|
|
39
|
+
Enabled: false
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yudai Takada
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
# Sidekiq Vigil
|
|
2
|
+
|
|
3
|
+
Sidekiq Vigil is battery-included monitoring and alerting for Sidekiq. It runs against the Redis you already use, evaluates twelve health checks, keeps an alert lifecycle across restarts, and sends firing, ongoing, and recovery notifications.
|
|
4
|
+
|
|
5
|
+
[](https://github.com/ydah/sidekiq-vigil/actions/workflows/main.yml)
|
|
6
|
+
|
|
7
|
+
## Requirements
|
|
8
|
+
|
|
9
|
+
- Ruby 3.2 or newer
|
|
10
|
+
- Sidekiq 7.x or 8.x
|
|
11
|
+
- Redis 7 or a Sidekiq-compatible Redis implementation
|
|
12
|
+
|
|
13
|
+
Sidekiq is the only runtime dependency.
|
|
14
|
+
|
|
15
|
+
## Five-minute quick start
|
|
16
|
+
|
|
17
|
+
Add the gem from Git until the first RubyGems release:
|
|
18
|
+
|
|
19
|
+
```ruby
|
|
20
|
+
gem "sidekiq-vigil", github: "ydah/sidekiq-vigil"
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Generate an initializer in Rails:
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
bundle install
|
|
27
|
+
bundle exec rails generate sidekiq_vigil:install
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Or create a pure Ruby initializer:
|
|
31
|
+
|
|
32
|
+
```ruby
|
|
33
|
+
require "sidekiq_vigil"
|
|
34
|
+
|
|
35
|
+
SidekiqVigil.configure do |config|
|
|
36
|
+
config.key_prefix = "myapp"
|
|
37
|
+
config.redis = { url: ENV["REDIS_URL"] }
|
|
38
|
+
|
|
39
|
+
config.notifier :slack,
|
|
40
|
+
webhook_url: ENV["SLACK_WEBHOOK_DEFAULT"],
|
|
41
|
+
routes: { critical: ENV["SLACK_WEBHOOK_INCIDENTS"] }.compact,
|
|
42
|
+
mention: { critical: "<!here>" },
|
|
43
|
+
web_ui_url: "https://example.com/sidekiq"
|
|
44
|
+
config.notifier :log
|
|
45
|
+
|
|
46
|
+
config.check :queue_latency, warn: 60, critical: 300
|
|
47
|
+
config.check :queue_size, warn: 1_000, critical: 10_000
|
|
48
|
+
end
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Start Sidekiq normally. The server middleware, Reporter, and leader-elected Checker are installed automatically. Outside `production`, external notification is disabled unless `config.enabled = true` is assigned explicitly; Log remains available.
|
|
52
|
+
|
|
53
|
+
Test every configured notifier before relying on it:
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
bundle exec vigil --config config/sidekiq_vigil.rb test-notify
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Recommended production topology
|
|
60
|
+
|
|
61
|
+
Use Embedded mode for collection and checks, expose `SidekiqVigil::HealthApp`, and have an external uptime monitor call `/healthz`. Embedded monitoring cannot run when every Sidekiq process is dead; the endpoint detects that condition because an expired snapshot returns 503.
|
|
62
|
+
|
|
63
|
+
```ruby
|
|
64
|
+
# config.ru
|
|
65
|
+
require "sidekiq_vigil"
|
|
66
|
+
|
|
67
|
+
storage = SidekiqVigil.build_storage
|
|
68
|
+
health = SidekiqVigil::HealthApp.new(
|
|
69
|
+
storage: storage,
|
|
70
|
+
interval: SidekiqVigil.config.interval
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
# Authentication is intentionally supplied by the application.
|
|
74
|
+
use Rack::Auth::Basic do |username, password|
|
|
75
|
+
Rack::Utils.secure_compare(username, ENV.fetch("VIGIL_HEALTH_USER")) &&
|
|
76
|
+
Rack::Utils.secure_compare(password, ENV.fetch("VIGIL_HEALTH_PASSWORD"))
|
|
77
|
+
end
|
|
78
|
+
run health
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`GET /healthz` returns 200 only when every result is OK and the snapshot is at most `interval × 2` old. `GET /status.json` includes check results, freshness, and age. Both return 503 when unhealthy or unavailable.
|
|
82
|
+
|
|
83
|
+
## Execution modes
|
|
84
|
+
|
|
85
|
+
| Mode | Command/integration | Collection | Detects total Sidekiq outage |
|
|
86
|
+
|---|---|---:|---:|
|
|
87
|
+
| Embedded (default) | Require the gem in Sidekiq | Reporter + leader Checker | Through stale external `/healthz` probe |
|
|
88
|
+
| Standalone | `vigil --config config/vigil.rb` | Checker only | Yes |
|
|
89
|
+
| Oneshot | `vigil --config config/vigil.rb check --once` | One Checker cycle | Yes, when scheduled externally |
|
|
90
|
+
|
|
91
|
+
Standalone configuration is plain Ruby and must not depend on Rails constants.
|
|
92
|
+
|
|
93
|
+
```sh
|
|
94
|
+
bundle exec vigil --config config/vigil.rb status
|
|
95
|
+
bundle exec vigil --config config/vigil.rb mute 30m --reason "database maintenance"
|
|
96
|
+
bundle exec vigil --config config/vigil.rb unmute
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`TERM` and `INT` stop Standalone mode gracefully and release its leader lock.
|
|
100
|
+
|
|
101
|
+
## Check catalog
|
|
102
|
+
|
|
103
|
+
| Check | Default behavior | Important options |
|
|
104
|
+
|---|---|---|
|
|
105
|
+
| `queue_latency` | Warn 60s, critical 300s | `queues`, `per_queue` |
|
|
106
|
+
| `queue_size` | Warn 1,000, critical 10,000 | `queues`, `per_queue` |
|
|
107
|
+
| `retry_set` | Warn 100, critical 1,000 | `growth_only` |
|
|
108
|
+
| `dead_set` | Warn 1, critical 50 | `growth_only` |
|
|
109
|
+
| `process_alive` | Require at least one active process | `min_processes`, `quiet_threshold` |
|
|
110
|
+
| `utilization` | Warn 85%, critical 95% after 300s | `sustained` |
|
|
111
|
+
| `failure_rate` | Warn 5%, critical 20%, minimum 20 attempts | `window`, `min_samples` |
|
|
112
|
+
| `stuck_jobs` | Critical after 30 minutes | `threshold` |
|
|
113
|
+
| `memory` | Warn 1,500 MiB, critical 2,500 MiB | `warn_mb`, `critical_mb` |
|
|
114
|
+
| `redis_health` | PING latency and maxmemory usage | `latency_ms`, `memory_pct` |
|
|
115
|
+
| `scheduled_backlog` | Warn on jobs overdue by five minutes | `overdue` |
|
|
116
|
+
| `throughput_anomaly` | Disabled by default | `drop_pct`, `baseline_days`, `min_samples` |
|
|
117
|
+
|
|
118
|
+
All checks use Sidekiq's public `sidekiq/api` objects. A check exception becomes an `error` result; it does not terminate Sidekiq or the Checker loop. Custom checks subclass `SidekiqVigil::Check::Base` and return one `SidekiqVigil::Result` or an array.
|
|
119
|
+
|
|
120
|
+
Failure rate counts execution attempts, so a job that fails and retries can contribute more than one failure. Stuck-job duration assumes synchronized host clocks (NTP).
|
|
121
|
+
|
|
122
|
+
## Full configuration reference
|
|
123
|
+
|
|
124
|
+
```ruby
|
|
125
|
+
SidekiqVigil.configure do |config|
|
|
126
|
+
config.enabled = true
|
|
127
|
+
config.interval = 30
|
|
128
|
+
config.flush_interval = 10
|
|
129
|
+
config.key_prefix = "myapp"
|
|
130
|
+
config.timezone = "Asia/Tokyo"
|
|
131
|
+
config.redis = {
|
|
132
|
+
url: ENV["REDIS_URL"],
|
|
133
|
+
pool_size: 5,
|
|
134
|
+
pool_timeout: 1
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
config.notifier :slack,
|
|
138
|
+
webhook_url: ENV["SLACK_WEBHOOK_DEFAULT"],
|
|
139
|
+
routes: { critical: ENV["SLACK_WEBHOOK_INCIDENTS"] },
|
|
140
|
+
mention: { critical: "<!subteam^S012345>" },
|
|
141
|
+
web_ui_url: "https://example.com/sidekiq"
|
|
142
|
+
config.notifier :webhook, url: "https://events.example.com/sidekiq"
|
|
143
|
+
config.notifier :log
|
|
144
|
+
|
|
145
|
+
config.alerting do |alerting|
|
|
146
|
+
alerting.pending_cycles = 2
|
|
147
|
+
alerting.cooldown = 600
|
|
148
|
+
alerting.resolve_notice = true
|
|
149
|
+
alerting.flap_window = 120
|
|
150
|
+
alerting.flap_threshold = 4
|
|
151
|
+
alerting.escalate_after = 3
|
|
152
|
+
alerting.group_threshold = 5
|
|
153
|
+
alerting.group_top_n = 5
|
|
154
|
+
alerting.mutes = [
|
|
155
|
+
{ cron: "0 3 * * 0", duration: 3_600, reason: "weekly maintenance" }
|
|
156
|
+
]
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Slack Incoming Webhooks have a fixed channel. Severity routing therefore selects different URLs; Sidekiq Vigil does not attempt to override `channel` in a payload. Mentions must use `<@U…>`, `<!subteam^S…>`, or `<!here>` syntax. Plain text such as `@oncall` fails configuration validation. Secret URLs are filtered from notifier inspection and error logging.
|
|
162
|
+
|
|
163
|
+
The generic webhook contract and examples are documented in [docs/webhook_schema.md](docs/webhook_schema.md).
|
|
164
|
+
|
|
165
|
+
The five Slack Block Kit shapes are contract-tested against JSON previews for [critical](spec/fixtures/slack/critical.json), [warn](spec/fixtures/slack/warn.json), [resolved](spec/fixtures/slack/resolved.json), [still-firing](spec/fixtures/slack/still_firing.json), and [digest](spec/fixtures/slack/digest.json). A real Slack screenshot must come from the manual pre-release check; generated imagery is not used as release evidence.
|
|
166
|
+
|
|
167
|
+
## Alert lifecycle
|
|
168
|
+
|
|
169
|
+
Each `check_name + target` moves through `OK → PENDING → FIRING → RESOLVED → OK`. State survives leader changes and process restarts.
|
|
170
|
+
|
|
171
|
+
- `pending_cycles` removes single-cycle noise.
|
|
172
|
+
- `cooldown` controls ongoing notifications.
|
|
173
|
+
- `escalate_after` upgrades sustained warnings to critical.
|
|
174
|
+
- `flap_threshold` transitions inside `flap_window` emit one flapping event and hold refiring until the window closes.
|
|
175
|
+
- history retains the latest 30 values for 24 hours.
|
|
176
|
+
- targets that disappear are pruned every cycle.
|
|
177
|
+
- more than `group_threshold` events in one cycle become one digest with severity counts and `group_top_n` details.
|
|
178
|
+
- mute windows continue state transitions but suppress delivery; unmute reports current firing state once.
|
|
179
|
+
|
|
180
|
+
Mute cron expressions use the standard five fields and support lists (`1,3`), ranges (`1-5`), and steps (`*/10`). When both day-of-month and weekday are restricted, standard cron OR semantics apply.
|
|
181
|
+
|
|
182
|
+
All Redis keys start with `vigil:<key_prefix>:`. The Storage API rejects non-positive or missing TTLs except for the explicitly managed `alerts` hash. The complete, contract-tested key list is in [docs/redis_keys.md](docs/redis_keys.md).
|
|
183
|
+
|
|
184
|
+
## Existing tools
|
|
185
|
+
|
|
186
|
+
| Tool | Primary purpose | Built-in lifecycle/notifications | Additional stack |
|
|
187
|
+
|---|---|---:|---:|
|
|
188
|
+
| yabeda-sidekiq / Prometheus exporters | Metrics export | No | Prometheus/Grafana/Alertmanager |
|
|
189
|
+
| Sidekiq Alive | Kubernetes liveness | No | External probe |
|
|
190
|
+
| sidekiq-job_alert | Queue/dead Slack alerts | Limited | No |
|
|
191
|
+
| Sidekiq Web metrics | Execution exploration | No | No |
|
|
192
|
+
| Sidekiq Vigil | Checks, lifecycle, mute/group/recovery | Yes | No |
|
|
193
|
+
|
|
194
|
+
## Known limitations
|
|
195
|
+
|
|
196
|
+
- During complete Redis failure, each process loses leader coordination and direct `redis_health` notifications can be duplicated. Recovery may also cause another notification. Use an external `/healthz` probe as the primary total-outage signal.
|
|
197
|
+
- Throughput anomaly baseline uses the previous seven same-time windows and does not model weekday/weekend seasonality. It is opt-in and has a minimum-sample guard.
|
|
198
|
+
- Stuck-job detection compares timestamps from different hosts and requires synchronized clocks.
|
|
199
|
+
- RSS collection supports Linux `/proc` and macOS `ps`; the memory check is skipped on Windows.
|
|
200
|
+
- Sidekiq Pro and Enterprise use only the same public APIs, but the combinations are not yet separately certified.
|
|
201
|
+
- Slack Bot Token mode, thread tracking, weekday baselines, and Prometheus output are planned after v1.
|
|
202
|
+
|
|
203
|
+
## Development
|
|
204
|
+
|
|
205
|
+
```sh
|
|
206
|
+
bundle install
|
|
207
|
+
redis-server --port 16379 --save "" --appendonly no
|
|
208
|
+
VIGIL_REDIS_URL=redis://127.0.0.1:16379/15 bundle exec rake
|
|
209
|
+
bundle exec appraisal sidekiq_7 rspec
|
|
210
|
+
bundle exec appraisal sidekiq_8 rspec
|
|
211
|
+
bundle exec ruby benchmark/middleware_overhead.rb
|
|
212
|
+
bundle exec rake build
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The benchmark compares middleware-on and middleware-off in the same process, writes a JSON artifact, and only warns when the relative run has unusually high overhead.
|
|
216
|
+
|
|
217
|
+
## FAQ
|
|
218
|
+
|
|
219
|
+
**Does the middleware call Redis for every job?**
|
|
220
|
+
|
|
221
|
+
No. It updates a mutex-protected in-process counter. Reporter flushes aggregates in a Redis pipeline.
|
|
222
|
+
|
|
223
|
+
**Can I run Embedded and Standalone together?**
|
|
224
|
+
|
|
225
|
+
Yes. The Redis leader lock prevents duplicate normal checks. A configuration digest warning appears when process configurations differ.
|
|
226
|
+
|
|
227
|
+
**Why did Slack not receive a development notification?**
|
|
228
|
+
|
|
229
|
+
External notifiers are safe by default outside production. Explicitly assign `config.enabled = true`, then run `vigil test-notify`.
|
|
230
|
+
|
|
231
|
+
**Can I keep using yabeda or an APM?**
|
|
232
|
+
|
|
233
|
+
Yes. Sidekiq Vigil does not replace or modify Sidekiq's metrics subsystem.
|
|
234
|
+
|
|
235
|
+
## License
|
|
236
|
+
|
|
237
|
+
MIT. See [LICENSE.txt](LICENSE.txt).
|
data/Rakefile
ADDED
data/docs/redis_keys.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Redis key contract
|
|
2
|
+
|
|
3
|
+
Every key is namespaced below `vigil:<key_prefix>:`. Storage writes require a positive TTL; the `alerts` hash is the only persistent exception and is explicitly pruned by `Alert::Manager`.
|
|
4
|
+
|
|
5
|
+
The first column of this table is contract-tested against `SidekiqVigil::Storage::KEY_CATALOG`.
|
|
6
|
+
|
|
7
|
+
| Key suffix | Type | Content | TTL | Owner |
|
|
8
|
+
|---|---|---|---|---|
|
|
9
|
+
| `stats:{yyyymmddHHMM}` | hash | Minute counters for processed/failed jobs and worker classes | 8 days | Reporter |
|
|
10
|
+
| `exec:{queue}` | hash | Execution count/sum/max by queue | 1 hour | Reporter |
|
|
11
|
+
| `alerts` | hash | Serialized alert lifecycle state | Explicitly managed; stale targets are pruned | Alert::Manager |
|
|
12
|
+
| `history:{alert_id}` | list | Latest 30 observed values for a single alert | 24 hours | Alert::Manager |
|
|
13
|
+
| `snapshot` | string | Timestamped check results and alert states | `interval × 4` | Checker |
|
|
14
|
+
| `leader` | string | Leader-election token | `interval × 3` | LeaderElection |
|
|
15
|
+
| `mem:{process_id}` | string | Reporter RSS in KiB | `flush_interval × 3` | Reporter |
|
|
16
|
+
| `config_digest` | hash | Configuration digest by process | 1 hour | Reporter |
|
|
17
|
+
| `mute` | string | Manual mute reason and expiration | Requested duration | Alert::Mute |
|
|
18
|
+
| `check_state:{check-specific-suffix}` | string | Growth, quiet-process, or sustained-condition state | Check-specific positive TTL | Checks |
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/ydah/sidekiq-vigil/blob/main/docs/webhook_event.schema.json",
|
|
4
|
+
"title": "Sidekiq Vigil webhook event",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["schema_version", "event", "alert_id", "timestamp"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schema_version": { "const": "1.0" },
|
|
9
|
+
"event": {
|
|
10
|
+
"enum": ["firing", "still_firing", "resolved", "escalated", "flapping", "unmuted", "digest"]
|
|
11
|
+
},
|
|
12
|
+
"alert_id": { "type": "string", "minLength": 1 },
|
|
13
|
+
"timestamp": { "type": "string", "format": "date-time" },
|
|
14
|
+
"alert": {
|
|
15
|
+
"type": "object",
|
|
16
|
+
"required": ["check_name", "target", "severity", "metadata"],
|
|
17
|
+
"properties": {
|
|
18
|
+
"check_name": { "type": "string" },
|
|
19
|
+
"target": { "type": "string" },
|
|
20
|
+
"severity": { "enum": ["ok", "warn", "critical", "error"] },
|
|
21
|
+
"value": {},
|
|
22
|
+
"threshold": {},
|
|
23
|
+
"message": { "type": ["string", "null"] },
|
|
24
|
+
"metadata": { "type": "object" }
|
|
25
|
+
},
|
|
26
|
+
"additionalProperties": false
|
|
27
|
+
},
|
|
28
|
+
"state": { "type": "object" },
|
|
29
|
+
"history": { "type": "array" },
|
|
30
|
+
"counts": { "type": "object" },
|
|
31
|
+
"alerts": { "type": "array" }
|
|
32
|
+
},
|
|
33
|
+
"allOf": [
|
|
34
|
+
{
|
|
35
|
+
"if": { "properties": { "event": { "const": "digest" } } },
|
|
36
|
+
"then": { "required": ["counts", "alerts"] },
|
|
37
|
+
"else": { "required": ["alert", "state", "history"] }
|
|
38
|
+
}
|
|
39
|
+
],
|
|
40
|
+
"additionalProperties": false
|
|
41
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Webhook payload schema
|
|
2
|
+
|
|
3
|
+
The generic webhook notifier sends one JSON object per alert event. The normative JSON Schema is [`webhook_event.schema.json`](webhook_event.schema.json).
|
|
4
|
+
|
|
5
|
+
## Compatibility
|
|
6
|
+
|
|
7
|
+
- `schema_version` is currently `1.0`.
|
|
8
|
+
- New optional properties may be added in a minor release.
|
|
9
|
+
- Removing a property, changing its type, or changing event semantics requires a schema major-version change.
|
|
10
|
+
- Receivers should ignore unknown optional properties.
|
|
11
|
+
|
|
12
|
+
## Regular event
|
|
13
|
+
|
|
14
|
+
```json
|
|
15
|
+
{
|
|
16
|
+
"schema_version": "1.0",
|
|
17
|
+
"event": "firing",
|
|
18
|
+
"alert_id": "queue_size:critical",
|
|
19
|
+
"timestamp": "2026-07-28T12:00:00Z",
|
|
20
|
+
"alert": {
|
|
21
|
+
"check_name": "queue_size",
|
|
22
|
+
"target": "critical",
|
|
23
|
+
"severity": "critical",
|
|
24
|
+
"value": 250,
|
|
25
|
+
"threshold": 200,
|
|
26
|
+
"message": "250 jobs waiting",
|
|
27
|
+
"metadata": {}
|
|
28
|
+
},
|
|
29
|
+
"state": {
|
|
30
|
+
"status": "firing",
|
|
31
|
+
"cycles": 2,
|
|
32
|
+
"first_seen_at": 1785239970.0,
|
|
33
|
+
"last_notified_at": 1785240000.0,
|
|
34
|
+
"last_transition_at": 1785240000.0,
|
|
35
|
+
"severity": "critical",
|
|
36
|
+
"value": 250,
|
|
37
|
+
"threshold": 200,
|
|
38
|
+
"message": "250 jobs waiting",
|
|
39
|
+
"suppressed": false,
|
|
40
|
+
"flap_notified": false,
|
|
41
|
+
"flapping_until": null,
|
|
42
|
+
"transition_timestamps": [1785240000.0]
|
|
43
|
+
},
|
|
44
|
+
"history": [
|
|
45
|
+
{ "timestamp": 1785239970.0, "value": 190, "severity": "warn" },
|
|
46
|
+
{ "timestamp": 1785240000.0, "value": 250, "severity": "critical" }
|
|
47
|
+
]
|
|
48
|
+
}
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Digest event
|
|
52
|
+
|
|
53
|
+
A digest replaces individual notifications when the number of events in one checker cycle is greater than `group_threshold`. It contains severity counts and the highest-severity alerts.
|
|
54
|
+
|
|
55
|
+
```json
|
|
56
|
+
{
|
|
57
|
+
"schema_version": "1.0",
|
|
58
|
+
"event": "digest",
|
|
59
|
+
"alert_id": "digest",
|
|
60
|
+
"timestamp": "2026-07-28T12:00:00Z",
|
|
61
|
+
"counts": { "critical": 3, "warn": 4 },
|
|
62
|
+
"alerts": []
|
|
63
|
+
}
|
|
64
|
+
```
|
data/exe/vigil
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
begin
|
|
6
|
+
require "rails/generators"
|
|
7
|
+
|
|
8
|
+
module SidekiqVigil
|
|
9
|
+
module Generators
|
|
10
|
+
class InstallGenerator < Rails::Generators::Base
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
def copy_initializer
|
|
14
|
+
template "sidekiq_vigil.rb", "config/initializers/sidekiq_vigil.rb"
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
rescue LoadError
|
|
20
|
+
module SidekiqVigil
|
|
21
|
+
module Generators
|
|
22
|
+
class InstallGenerator
|
|
23
|
+
TEMPLATE = File.expand_path("templates/sidekiq_vigil.rb", __dir__)
|
|
24
|
+
|
|
25
|
+
def self.install(destination)
|
|
26
|
+
FileUtils.mkdir_p(File.dirname(destination))
|
|
27
|
+
FileUtils.cp(TEMPLATE, destination)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq_vigil"
|
|
4
|
+
|
|
5
|
+
SidekiqVigil.configure do |config|
|
|
6
|
+
# Monitoring is enabled by default. External notification is enabled by
|
|
7
|
+
# default only in production; set this explicitly to opt in elsewhere.
|
|
8
|
+
config.enabled = ENV.fetch("VIGIL_ENABLED") == "true" if ENV.key?("VIGIL_ENABLED")
|
|
9
|
+
config.interval = Integer(ENV.fetch("VIGIL_INTERVAL", "30"))
|
|
10
|
+
config.flush_interval = Integer(ENV.fetch("VIGIL_FLUSH_INTERVAL", "10"))
|
|
11
|
+
config.key_prefix = ENV.fetch("VIGIL_KEY_PREFIX", "myapp")
|
|
12
|
+
config.timezone = ENV.fetch("VIGIL_TIMEZONE", "UTC")
|
|
13
|
+
redis_url = ENV.fetch("REDIS_URL", nil)
|
|
14
|
+
config.redis = { url: redis_url } if redis_url
|
|
15
|
+
|
|
16
|
+
config.check :queue_latency, warn: 60, critical: 300
|
|
17
|
+
config.check :queue_size, warn: 1_000, critical: 10_000
|
|
18
|
+
config.check :retry_set, warn: 100, critical: 1_000
|
|
19
|
+
config.check :dead_set, warn: 1, critical: 50, growth_only: true
|
|
20
|
+
config.check :process_alive, min_processes: 1
|
|
21
|
+
config.check :utilization, warn: 0.85, critical: 0.95, sustained: 300
|
|
22
|
+
config.check :failure_rate, warn: 0.05, critical: 0.20, window: 300, min_samples: 20
|
|
23
|
+
config.check :stuck_jobs, threshold: 1_800
|
|
24
|
+
config.check :memory, warn_mb: 1_500, critical_mb: 2_500
|
|
25
|
+
config.check :redis_health, latency_ms: 100, memory_pct: 0.9
|
|
26
|
+
config.check :scheduled_backlog, overdue: 300
|
|
27
|
+
# Opt in after enough history has accumulated:
|
|
28
|
+
# config.check :throughput_anomaly, drop_pct: 0.5, baseline_days: 7
|
|
29
|
+
|
|
30
|
+
slack_webhook = ENV.fetch("SLACK_WEBHOOK_DEFAULT", nil)
|
|
31
|
+
if slack_webhook
|
|
32
|
+
config.notifier :slack,
|
|
33
|
+
webhook_url: slack_webhook,
|
|
34
|
+
routes: { critical: ENV.fetch("SLACK_WEBHOOK_INCIDENTS", nil) }.compact,
|
|
35
|
+
mention: { critical: ENV.fetch("SLACK_CRITICAL_MENTION", nil) }.compact,
|
|
36
|
+
web_ui_url: ENV.fetch("SIDEKIQ_WEB_UI_URL", nil)
|
|
37
|
+
end
|
|
38
|
+
config.notifier :log
|
|
39
|
+
|
|
40
|
+
config.alerting do |alerting|
|
|
41
|
+
alerting.pending_cycles = 2
|
|
42
|
+
alerting.cooldown = 600
|
|
43
|
+
alerting.resolve_notice = true
|
|
44
|
+
alerting.flap_window = 120
|
|
45
|
+
alerting.flap_threshold = 4
|
|
46
|
+
alerting.group_threshold = 5
|
|
47
|
+
alerting.group_top_n = 5
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SidekiqVigil
|
|
4
|
+
module Alert
|
|
5
|
+
class Cron
|
|
6
|
+
Field = Data.define(:values, :wildcard)
|
|
7
|
+
|
|
8
|
+
FIELD_RANGES = [
|
|
9
|
+
(0..59),
|
|
10
|
+
(0..23),
|
|
11
|
+
(1..31),
|
|
12
|
+
(1..12),
|
|
13
|
+
(0..7)
|
|
14
|
+
].freeze
|
|
15
|
+
|
|
16
|
+
def initialize(expression)
|
|
17
|
+
fields = expression.to_s.split
|
|
18
|
+
raise ConfigError, "mute cron must contain five fields" unless fields.length == FIELD_RANGES.length
|
|
19
|
+
|
|
20
|
+
@fields = fields.zip(FIELD_RANGES).map { |field, range| parse_field(field, range) }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def match?(time)
|
|
24
|
+
minute, hour, day, month, weekday = fields
|
|
25
|
+
fixed_fields = [[minute, time.min], [hour, time.hour], [month, time.month]]
|
|
26
|
+
return false unless fixed_fields.all? { |field, value| field.values.include?(value) }
|
|
27
|
+
|
|
28
|
+
day_matches = day.values.include?(time.day)
|
|
29
|
+
weekday_matches = weekday.values.include?(time.wday)
|
|
30
|
+
return day_matches || weekday_matches unless day.wildcard || weekday.wildcard
|
|
31
|
+
|
|
32
|
+
day_matches && weekday_matches
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
attr_reader :fields
|
|
38
|
+
|
|
39
|
+
def parse_field(source, range)
|
|
40
|
+
wildcard = source.start_with?("*")
|
|
41
|
+
values = source.split(",").flat_map { |part| expand_part(part, range) }.map do |value|
|
|
42
|
+
normalize_weekday(value, range)
|
|
43
|
+
end
|
|
44
|
+
raise ConfigError, "mute cron field #{source.inspect} selects no values" if values.empty?
|
|
45
|
+
|
|
46
|
+
Field.new(values: values.uniq.freeze, wildcard:)
|
|
47
|
+
rescue ArgumentError
|
|
48
|
+
raise ConfigError, "invalid mute cron field #{source.inspect}"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def expand_part(part, range)
|
|
52
|
+
base, raw_step = split_part(part)
|
|
53
|
+
step = parse_step(raw_step)
|
|
54
|
+
values = base_values(base, range, stepped: !raw_step.nil?)
|
|
55
|
+
values.each_with_index.filter_map { |value, index| value if (index % step).zero? }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def split_part(part)
|
|
59
|
+
base, raw_step, extra = part.split("/", 3)
|
|
60
|
+
raise ArgumentError if base.nil? || base.empty? || extra
|
|
61
|
+
|
|
62
|
+
[base, raw_step]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def parse_step(raw_step)
|
|
66
|
+
return 1 unless raw_step
|
|
67
|
+
|
|
68
|
+
step = Integer(raw_step, 10)
|
|
69
|
+
raise ArgumentError unless step.positive?
|
|
70
|
+
|
|
71
|
+
step
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def base_values(base, range, stepped:)
|
|
75
|
+
return range.to_a if base == "*"
|
|
76
|
+
|
|
77
|
+
if base.include?("-")
|
|
78
|
+
first, last, extra = base.split("-", 3)
|
|
79
|
+
raise ArgumentError if extra
|
|
80
|
+
|
|
81
|
+
first = bounded_integer(first, range)
|
|
82
|
+
last = bounded_integer(last, range)
|
|
83
|
+
raise ArgumentError if first > last
|
|
84
|
+
|
|
85
|
+
return (first..last).to_a
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
first = bounded_integer(base, range)
|
|
89
|
+
return (first..range.end).to_a if stepped
|
|
90
|
+
|
|
91
|
+
[first]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def bounded_integer(value, range)
|
|
95
|
+
integer = Integer(value, 10)
|
|
96
|
+
raise ArgumentError unless range.cover?(integer)
|
|
97
|
+
|
|
98
|
+
integer
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def normalize_weekday(value, range)
|
|
102
|
+
range.end == 7 && value == 7 ? 0 : value
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|