solid_queue_heroku_autoscaler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +128 -0
- data/LICENSE.txt +21 -0
- data/README.md +474 -0
- data/lib/generators/solid_queue_heroku_autoscaler/install_generator.rb +21 -0
- data/lib/generators/solid_queue_heroku_autoscaler/migration_generator.rb +29 -0
- data/lib/generators/solid_queue_heroku_autoscaler/templates/README +41 -0
- data/lib/generators/solid_queue_heroku_autoscaler/templates/create_solid_queue_autoscaler_state.rb.erb +15 -0
- data/lib/generators/solid_queue_heroku_autoscaler/templates/initializer.rb +52 -0
- data/lib/solid_queue_heroku_autoscaler/adapters/base.rb +102 -0
- data/lib/solid_queue_heroku_autoscaler/adapters/heroku.rb +93 -0
- data/lib/solid_queue_heroku_autoscaler/adapters/kubernetes.rb +158 -0
- data/lib/solid_queue_heroku_autoscaler/adapters.rb +57 -0
- data/lib/solid_queue_heroku_autoscaler/advisory_lock.rb +71 -0
- data/lib/solid_queue_heroku_autoscaler/autoscale_job.rb +71 -0
- data/lib/solid_queue_heroku_autoscaler/configuration.rb +217 -0
- data/lib/solid_queue_heroku_autoscaler/cooldown_tracker.rb +153 -0
- data/lib/solid_queue_heroku_autoscaler/decision_engine.rb +228 -0
- data/lib/solid_queue_heroku_autoscaler/errors.rb +44 -0
- data/lib/solid_queue_heroku_autoscaler/metrics.rb +172 -0
- data/lib/solid_queue_heroku_autoscaler/railtie.rb +149 -0
- data/lib/solid_queue_heroku_autoscaler/scaler.rb +227 -0
- data/lib/solid_queue_heroku_autoscaler/version.rb +5 -0
- data/lib/solid_queue_heroku_autoscaler.rb +106 -0
- metadata +169 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: d232e8e59b0f059186709890d2b293b9498cbee779620582e58540908af2ce49
|
|
4
|
+
data.tar.gz: 9fad9708b81ca955208c77095a1f395c513e89d3524cfdb7d30d8357fb4fe346
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: a61ccb5462fe1dfa0941c857aa1c60cf61a62e50338d13b122904b2d0208c3a9dd615a72dd320ceb124d6540b36453e070ca76657e66160b83a8aaeb04be622b
|
|
7
|
+
data.tar.gz: aec6a58d0243d07ef9ecc289b7dff1bbfa87dc8e031d2d92d44a497ef48efcf7d914399611b938be372ad5fca578a58cb5cc34a50b3a6ff7a0aae24ae03c0cf7
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [0.1.0] - 2025-01-XX
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
#### Core Features
|
|
13
|
+
- **Metrics-based autoscaling** for Solid Queue workers on Heroku and Kubernetes
|
|
14
|
+
- **Queue depth monitoring** - Scale based on number of pending jobs
|
|
15
|
+
- **Latency monitoring** - Scale based on oldest job age
|
|
16
|
+
- **Throughput tracking** - Monitor jobs processed per minute
|
|
17
|
+
|
|
18
|
+
#### Scaling Strategies
|
|
19
|
+
- **Fixed scaling** - Add/remove a fixed number of workers per scaling event
|
|
20
|
+
- **Proportional scaling** - Scale workers proportionally based on load level
|
|
21
|
+
|
|
22
|
+
#### Multi-Worker Support
|
|
23
|
+
- **Named configurations** - Configure multiple worker types independently
|
|
24
|
+
- **Per-worker cooldowns** - Each worker type has its own cooldown tracking
|
|
25
|
+
- **Unique advisory locks** - Parallel scaling of different worker types
|
|
26
|
+
- **Queue filtering** - Assign specific queues to each worker configuration
|
|
27
|
+
|
|
28
|
+
#### Infrastructure Adapters
|
|
29
|
+
- **Heroku adapter** - Scale dynos via Heroku Platform API
|
|
30
|
+
- **Kubernetes adapter** - Scale deployments via Kubernetes API
|
|
31
|
+
- **Pluggable architecture** - Easy to add custom adapters
|
|
32
|
+
|
|
33
|
+
#### Safety Features
|
|
34
|
+
- **PostgreSQL advisory locks** - Singleton execution across multiple dynos
|
|
35
|
+
- **Configurable cooldowns** - Prevent rapid scaling oscillations
|
|
36
|
+
- **Separate scale-up/down cooldowns** - Fine-tune scaling behavior
|
|
37
|
+
- **Min/max worker limits** - Prevent over/under-provisioning
|
|
38
|
+
- **Dry-run mode** - Test scaling decisions without making changes
|
|
39
|
+
|
|
40
|
+
#### Persistence
|
|
41
|
+
- **Database-backed cooldown tracking** - Survives dyno restarts
|
|
42
|
+
- **Migration generator** - Easy setup of state table
|
|
43
|
+
- **Fallback to in-memory** - Works without migration
|
|
44
|
+
|
|
45
|
+
#### Rails Integration
|
|
46
|
+
- **Railtie with rake tasks** - `scale`, `scale_all`, `metrics`, `formation`, `cooldown`, `reset_cooldown`, `workers`
|
|
47
|
+
- **Configuration initializer generator** - Quick setup
|
|
48
|
+
- **ActiveJob integration** - `AutoscaleJob` for recurring execution
|
|
49
|
+
- **Solid Queue recurring job support** - Run autoscaler on schedule
|
|
50
|
+
|
|
51
|
+
#### Developer Experience
|
|
52
|
+
- **Comprehensive test suite** - 356 RSpec examples
|
|
53
|
+
- **RuboCop configuration** - Clean, consistent code style
|
|
54
|
+
- **GitHub Actions CI** - Automated testing on Ruby 3.1, 3.2, 3.3
|
|
55
|
+
- **Detailed logging** - Track all scaling decisions and actions
|
|
56
|
+
|
|
57
|
+
### Configuration Options
|
|
58
|
+
|
|
59
|
+
```ruby
|
|
60
|
+
SolidQueueHerokuAutoscaler.configure(:worker_name) do |config|
|
|
61
|
+
# Heroku settings
|
|
62
|
+
config.heroku_api_key = ENV['HEROKU_API_KEY']
|
|
63
|
+
config.heroku_app_name = ENV['HEROKU_APP_NAME']
|
|
64
|
+
config.process_type = 'worker'
|
|
65
|
+
|
|
66
|
+
# Kubernetes settings (alternative to Heroku)
|
|
67
|
+
# config.adapter_class = SolidQueueHerokuAutoscaler::Adapters::Kubernetes
|
|
68
|
+
# config.kubernetes_deployment = 'my-worker'
|
|
69
|
+
# config.kubernetes_namespace = 'production'
|
|
70
|
+
|
|
71
|
+
# Worker limits
|
|
72
|
+
config.min_workers = 1
|
|
73
|
+
config.max_workers = 10
|
|
74
|
+
|
|
75
|
+
# Scaling strategy (:fixed or :proportional)
|
|
76
|
+
config.scaling_strategy = :fixed
|
|
77
|
+
|
|
78
|
+
# Scale-up thresholds
|
|
79
|
+
config.scale_up_queue_depth = 100
|
|
80
|
+
config.scale_up_latency_seconds = 300
|
|
81
|
+
config.scale_up_increment = 1
|
|
82
|
+
|
|
83
|
+
# Proportional scaling settings
|
|
84
|
+
config.scale_up_jobs_per_worker = 50
|
|
85
|
+
config.scale_up_latency_per_worker = 60
|
|
86
|
+
|
|
87
|
+
# Scale-down thresholds
|
|
88
|
+
config.scale_down_queue_depth = 10
|
|
89
|
+
config.scale_down_latency_seconds = 30
|
|
90
|
+
config.scale_down_decrement = 1
|
|
91
|
+
|
|
92
|
+
# Cooldowns
|
|
93
|
+
config.cooldown_seconds = 120
|
|
94
|
+
config.scale_up_cooldown_seconds = 60
|
|
95
|
+
config.scale_down_cooldown_seconds = 180
|
|
96
|
+
|
|
97
|
+
# Queue filtering
|
|
98
|
+
config.queues = ['default', 'mailers']
|
|
99
|
+
|
|
100
|
+
# Behavior
|
|
101
|
+
config.dry_run = false
|
|
102
|
+
config.enabled = true
|
|
103
|
+
|
|
104
|
+
# Custom table prefix for Solid Queue tables
|
|
105
|
+
config.table_prefix = 'solid_queue_'
|
|
106
|
+
end
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Usage Examples
|
|
110
|
+
|
|
111
|
+
```ruby
|
|
112
|
+
# Scale default worker
|
|
113
|
+
SolidQueueHerokuAutoscaler.scale!
|
|
114
|
+
|
|
115
|
+
# Scale specific worker type
|
|
116
|
+
SolidQueueHerokuAutoscaler.scale!(:critical_worker)
|
|
117
|
+
|
|
118
|
+
# Scale all configured workers
|
|
119
|
+
SolidQueueHerokuAutoscaler.scale_all!
|
|
120
|
+
|
|
121
|
+
# Get metrics for a worker
|
|
122
|
+
metrics = SolidQueueHerokuAutoscaler.metrics(:default)
|
|
123
|
+
|
|
124
|
+
# List registered workers
|
|
125
|
+
SolidQueueHerokuAutoscaler.registered_workers
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
[0.1.0]: https://github.com/reillyse/solid_queue_heroku_autoscaler/releases/tag/v0.1.0
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 reillyse
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
# Solid Queue Heroku Autoscaler
|
|
2
|
+
|
|
3
|
+
[](https://github.com/reillyse/solid_queue_heroku_autoscaler/actions/workflows/ci.yml)
|
|
4
|
+
[](https://badge.fury.io/rb/solid_queue_heroku_autoscaler)
|
|
5
|
+
|
|
6
|
+
A control plane for [Solid Queue](https://github.com/rails/solid_queue) that automatically scales worker processes based on queue metrics. Supports both **Heroku** and **Kubernetes** deployments.
|
|
7
|
+
|
|
8
|
+
## Features
|
|
9
|
+
|
|
10
|
+
- **Metrics-based scaling**: Scales based on queue depth, job latency, and throughput
|
|
11
|
+
- **Multiple scaling strategies**: Fixed increment or proportional scaling based on load
|
|
12
|
+
- **Multi-worker support**: Configure and scale different worker types independently
|
|
13
|
+
- **Platform adapters**: Native support for Heroku and Kubernetes
|
|
14
|
+
- **Singleton execution**: Uses PostgreSQL advisory locks to ensure only one autoscaler runs at a time
|
|
15
|
+
- **Safety features**: Cooldowns, min/max limits, dry-run mode
|
|
16
|
+
- **Rails integration**: Configuration via initializer, Railtie with rake tasks
|
|
17
|
+
- **Flexible execution**: Run as a recurring Solid Queue job or standalone
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
Add to your Gemfile:
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
gem 'solid_queue_heroku_autoscaler'
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Then run:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
bundle install
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### Database Setup (Recommended)
|
|
34
|
+
|
|
35
|
+
For persistent cooldown tracking that survives process restarts:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
rails generate solid_queue_heroku_autoscaler:migration
|
|
39
|
+
rails db:migrate
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
This creates a `solid_queue_autoscaler_state` table to store cooldown timestamps.
|
|
43
|
+
|
|
44
|
+
## Quick Start
|
|
45
|
+
|
|
46
|
+
### Basic Configuration (Single Worker)
|
|
47
|
+
|
|
48
|
+
Create an initializer at `config/initializers/solid_queue_autoscaler.rb`:
|
|
49
|
+
|
|
50
|
+
```ruby
|
|
51
|
+
SolidQueueHerokuAutoscaler.configure do |config|
|
|
52
|
+
# Platform: Heroku
|
|
53
|
+
config.adapter = :heroku
|
|
54
|
+
config.heroku_api_key = ENV['HEROKU_API_KEY']
|
|
55
|
+
config.heroku_app_name = ENV['HEROKU_APP_NAME']
|
|
56
|
+
config.process_type = 'worker'
|
|
57
|
+
|
|
58
|
+
# Worker limits
|
|
59
|
+
config.min_workers = 1
|
|
60
|
+
config.max_workers = 10
|
|
61
|
+
|
|
62
|
+
# Scaling thresholds
|
|
63
|
+
config.scale_up_queue_depth = 100
|
|
64
|
+
config.scale_up_latency_seconds = 300
|
|
65
|
+
config.scale_down_queue_depth = 10
|
|
66
|
+
config.scale_down_latency_seconds = 30
|
|
67
|
+
end
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Multi-Worker Configuration
|
|
71
|
+
|
|
72
|
+
Scale different worker types independently with named configurations:
|
|
73
|
+
|
|
74
|
+
```ruby
|
|
75
|
+
# Critical jobs worker - fast response, dedicated queue
|
|
76
|
+
SolidQueueHerokuAutoscaler.configure(:critical_worker) do |config|
|
|
77
|
+
config.adapter = :heroku
|
|
78
|
+
config.heroku_api_key = ENV['HEROKU_API_KEY']
|
|
79
|
+
config.heroku_app_name = ENV['HEROKU_APP_NAME']
|
|
80
|
+
config.process_type = 'critical_worker'
|
|
81
|
+
|
|
82
|
+
# Only monitor the critical queue
|
|
83
|
+
config.queues = ['critical']
|
|
84
|
+
|
|
85
|
+
# Aggressive scaling for critical jobs
|
|
86
|
+
config.min_workers = 2
|
|
87
|
+
config.max_workers = 20
|
|
88
|
+
config.scale_up_queue_depth = 10
|
|
89
|
+
config.scale_up_latency_seconds = 30
|
|
90
|
+
config.cooldown_seconds = 60
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Default worker - handles standard queues
|
|
94
|
+
SolidQueueHerokuAutoscaler.configure(:default_worker) do |config|
|
|
95
|
+
config.adapter = :heroku
|
|
96
|
+
config.heroku_api_key = ENV['HEROKU_API_KEY']
|
|
97
|
+
config.heroku_app_name = ENV['HEROKU_APP_NAME']
|
|
98
|
+
config.process_type = 'worker'
|
|
99
|
+
|
|
100
|
+
# Monitor default and mailers queues
|
|
101
|
+
config.queues = ['default', 'mailers']
|
|
102
|
+
|
|
103
|
+
# Conservative scaling for background jobs
|
|
104
|
+
config.min_workers = 1
|
|
105
|
+
config.max_workers = 10
|
|
106
|
+
config.scale_up_queue_depth = 100
|
|
107
|
+
config.scale_up_latency_seconds = 300
|
|
108
|
+
config.cooldown_seconds = 120
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Batch processing worker - handles long-running jobs
|
|
112
|
+
SolidQueueHerokuAutoscaler.configure(:batch_worker) do |config|
|
|
113
|
+
config.adapter = :heroku
|
|
114
|
+
config.heroku_api_key = ENV['HEROKU_API_KEY']
|
|
115
|
+
config.heroku_app_name = ENV['HEROKU_APP_NAME']
|
|
116
|
+
config.process_type = 'batch_worker'
|
|
117
|
+
|
|
118
|
+
config.queues = ['batch', 'imports', 'exports']
|
|
119
|
+
|
|
120
|
+
config.min_workers = 0
|
|
121
|
+
config.max_workers = 5
|
|
122
|
+
config.scale_up_queue_depth = 1 # Scale up when any batch job is queued
|
|
123
|
+
config.scale_down_queue_depth = 0
|
|
124
|
+
config.cooldown_seconds = 300
|
|
125
|
+
end
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## Platform Adapters
|
|
129
|
+
|
|
130
|
+
### Heroku Adapter (Default)
|
|
131
|
+
|
|
132
|
+
```ruby
|
|
133
|
+
SolidQueueHerokuAutoscaler.configure do |config|
|
|
134
|
+
config.adapter = :heroku
|
|
135
|
+
config.heroku_api_key = ENV['HEROKU_API_KEY']
|
|
136
|
+
config.heroku_app_name = ENV['HEROKU_APP_NAME']
|
|
137
|
+
config.process_type = 'worker' # Dyno type to scale
|
|
138
|
+
end
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Generate a Heroku API key:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
heroku authorizations:create -d "Solid Queue Autoscaler"
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### Kubernetes Adapter
|
|
148
|
+
|
|
149
|
+
```ruby
|
|
150
|
+
SolidQueueHerokuAutoscaler.configure do |config|
|
|
151
|
+
config.adapter = :kubernetes
|
|
152
|
+
config.kubernetes_namespace = ENV.fetch('KUBERNETES_NAMESPACE', 'default')
|
|
153
|
+
config.kubernetes_deployment = 'solid-queue-worker'
|
|
154
|
+
|
|
155
|
+
# Optional: Custom kubeconfig path (defaults to in-cluster config)
|
|
156
|
+
# config.kubernetes_config_path = '/path/to/kubeconfig'
|
|
157
|
+
end
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
The Kubernetes adapter uses the official `kubeclient` gem and supports:
|
|
161
|
+
- In-cluster service account authentication (recommended for production)
|
|
162
|
+
- External kubeconfig file authentication (useful for development)
|
|
163
|
+
|
|
164
|
+
## Configuration Reference
|
|
165
|
+
|
|
166
|
+
### Core Settings
|
|
167
|
+
|
|
168
|
+
| Option | Type | Default | Description |
|
|
169
|
+
|--------|------|---------|-------------|
|
|
170
|
+
| `adapter` | Symbol | `:heroku` | Platform adapter (`:heroku` or `:kubernetes`) |
|
|
171
|
+
| `enabled` | Boolean | `true` | Master switch to enable/disable autoscaling |
|
|
172
|
+
| `dry_run` | Boolean | `false` | Log decisions without making changes |
|
|
173
|
+
| `queues` | Array | `nil` | Queue names to monitor (nil = all queues) |
|
|
174
|
+
| `table_prefix` | String | `'solid_queue_'` | Solid Queue table name prefix |
|
|
175
|
+
|
|
176
|
+
### Worker Limits
|
|
177
|
+
|
|
178
|
+
| Option | Type | Default | Description |
|
|
179
|
+
|--------|------|---------|-------------|
|
|
180
|
+
| `min_workers` | Integer | `1` | Minimum workers to maintain |
|
|
181
|
+
| `max_workers` | Integer | `10` | Maximum workers allowed |
|
|
182
|
+
|
|
183
|
+
### Scale-Up Thresholds
|
|
184
|
+
|
|
185
|
+
Scaling up triggers when **ANY** threshold is exceeded:
|
|
186
|
+
|
|
187
|
+
| Option | Type | Default | Description |
|
|
188
|
+
|--------|------|---------|-------------|
|
|
189
|
+
| `scale_up_queue_depth` | Integer | `100` | Jobs in queue to trigger scale up |
|
|
190
|
+
| `scale_up_latency_seconds` | Integer | `300` | Oldest job age to trigger scale up |
|
|
191
|
+
| `scale_up_increment` | Integer | `1` | Workers to add (fixed strategy) |
|
|
192
|
+
|
|
193
|
+
### Scale-Down Thresholds
|
|
194
|
+
|
|
195
|
+
Scaling down triggers when **ALL** thresholds are met:
|
|
196
|
+
|
|
197
|
+
| Option | Type | Default | Description |
|
|
198
|
+
|--------|------|---------|-------------|
|
|
199
|
+
| `scale_down_queue_depth` | Integer | `10` | Jobs in queue threshold |
|
|
200
|
+
| `scale_down_latency_seconds` | Integer | `30` | Oldest job age threshold |
|
|
201
|
+
| `scale_down_decrement` | Integer | `1` | Workers to remove |
|
|
202
|
+
|
|
203
|
+
### Scaling Strategies
|
|
204
|
+
|
|
205
|
+
| Option | Type | Default | Description |
|
|
206
|
+
|--------|------|---------|-------------|
|
|
207
|
+
| `scaling_strategy` | Symbol | `:fixed` | `:fixed` or `:proportional` |
|
|
208
|
+
| `scale_up_jobs_per_worker` | Integer | `50` | Jobs per worker (proportional) |
|
|
209
|
+
| `scale_up_latency_per_worker` | Integer | `60` | Seconds per worker (proportional) |
|
|
210
|
+
| `scale_down_jobs_per_worker` | Integer | `50` | Jobs capacity per worker |
|
|
211
|
+
|
|
212
|
+
### Cooldowns
|
|
213
|
+
|
|
214
|
+
| Option | Type | Default | Description |
|
|
215
|
+
|--------|------|---------|-------------|
|
|
216
|
+
| `cooldown_seconds` | Integer | `120` | Default cooldown for both directions |
|
|
217
|
+
| `scale_up_cooldown_seconds` | Integer | `nil` | Override for scale-up cooldown |
|
|
218
|
+
| `scale_down_cooldown_seconds` | Integer | `nil` | Override for scale-down cooldown |
|
|
219
|
+
| `persist_cooldowns` | Boolean | `true` | Save cooldowns to database |
|
|
220
|
+
|
|
221
|
+
### Heroku-Specific
|
|
222
|
+
|
|
223
|
+
| Option | Type | Default | Description |
|
|
224
|
+
|--------|------|---------|-------------|
|
|
225
|
+
| `heroku_api_key` | String | `nil` | Heroku Platform API token |
|
|
226
|
+
| `heroku_app_name` | String | `nil` | Heroku app name |
|
|
227
|
+
| `process_type` | String | `'worker'` | Dyno type to scale |
|
|
228
|
+
|
|
229
|
+
### Kubernetes-Specific
|
|
230
|
+
|
|
231
|
+
| Option | Type | Default | Description |
|
|
232
|
+
|--------|------|---------|-------------|
|
|
233
|
+
| `kubernetes_namespace` | String | `'default'` | Kubernetes namespace |
|
|
234
|
+
| `kubernetes_deployment` | String | `nil` | Deployment name to scale |
|
|
235
|
+
| `kubernetes_config_path` | String | `nil` | Path to kubeconfig (optional) |
|
|
236
|
+
|
|
237
|
+
## Usage
|
|
238
|
+
|
|
239
|
+
### Running as a Solid Queue Recurring Job (Recommended)
|
|
240
|
+
|
|
241
|
+
Add to your `config/recurring.yml`:
|
|
242
|
+
|
|
243
|
+
```yaml
|
|
244
|
+
# Single worker configuration
|
|
245
|
+
autoscaler:
|
|
246
|
+
class: SolidQueueHerokuAutoscaler::AutoscaleJob
|
|
247
|
+
queue: autoscaler
|
|
248
|
+
schedule: every 30 seconds
|
|
249
|
+
|
|
250
|
+
# Or for multi-worker: scale all workers
|
|
251
|
+
autoscaler_all:
|
|
252
|
+
class: SolidQueueHerokuAutoscaler::AutoscaleJob
|
|
253
|
+
queue: autoscaler
|
|
254
|
+
schedule: every 30 seconds
|
|
255
|
+
args: [:all]
|
|
256
|
+
|
|
257
|
+
# Or scale specific worker types on different schedules
|
|
258
|
+
autoscaler_critical:
|
|
259
|
+
class: SolidQueueHerokuAutoscaler::AutoscaleJob
|
|
260
|
+
queue: autoscaler
|
|
261
|
+
schedule: every 15 seconds
|
|
262
|
+
args: [:critical_worker]
|
|
263
|
+
|
|
264
|
+
autoscaler_default:
|
|
265
|
+
class: SolidQueueHerokuAutoscaler::AutoscaleJob
|
|
266
|
+
queue: autoscaler
|
|
267
|
+
schedule: every 60 seconds
|
|
268
|
+
args: [:default_worker]
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
### Running via Rake Tasks
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
# Scale the default worker
|
|
275
|
+
bundle exec rake solid_queue_autoscaler:scale
|
|
276
|
+
|
|
277
|
+
# Scale a specific worker type
|
|
278
|
+
WORKER=critical_worker bundle exec rake solid_queue_autoscaler:scale
|
|
279
|
+
|
|
280
|
+
# Scale all configured workers
|
|
281
|
+
bundle exec rake solid_queue_autoscaler:scale_all
|
|
282
|
+
|
|
283
|
+
# List all registered worker configurations
|
|
284
|
+
bundle exec rake solid_queue_autoscaler:workers
|
|
285
|
+
|
|
286
|
+
# View metrics for default worker
|
|
287
|
+
bundle exec rake solid_queue_autoscaler:metrics
|
|
288
|
+
|
|
289
|
+
# View metrics for specific worker
|
|
290
|
+
WORKER=critical_worker bundle exec rake solid_queue_autoscaler:metrics
|
|
291
|
+
|
|
292
|
+
# View current formation
|
|
293
|
+
bundle exec rake solid_queue_autoscaler:formation
|
|
294
|
+
|
|
295
|
+
# Check cooldown status
|
|
296
|
+
bundle exec rake solid_queue_autoscaler:cooldown
|
|
297
|
+
|
|
298
|
+
# Reset cooldowns
|
|
299
|
+
bundle exec rake solid_queue_autoscaler:reset_cooldown
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
### Running Programmatically
|
|
303
|
+
|
|
304
|
+
```ruby
|
|
305
|
+
# Scale the default worker
|
|
306
|
+
result = SolidQueueHerokuAutoscaler.scale!
|
|
307
|
+
|
|
308
|
+
# Scale a specific worker type
|
|
309
|
+
result = SolidQueueHerokuAutoscaler.scale!(:critical_worker)
|
|
310
|
+
|
|
311
|
+
# Scale all configured workers
|
|
312
|
+
results = SolidQueueHerokuAutoscaler.scale_all!
|
|
313
|
+
|
|
314
|
+
# Get metrics for a specific worker
|
|
315
|
+
metrics = SolidQueueHerokuAutoscaler.metrics(:critical_worker)
|
|
316
|
+
puts "Queue depth: #{metrics.queue_depth}"
|
|
317
|
+
puts "Latency: #{metrics.oldest_job_age_seconds}s"
|
|
318
|
+
|
|
319
|
+
# Get current worker count
|
|
320
|
+
workers = SolidQueueHerokuAutoscaler.current_workers(:default_worker)
|
|
321
|
+
puts "Current workers: #{workers}"
|
|
322
|
+
|
|
323
|
+
# List all registered workers
|
|
324
|
+
SolidQueueHerokuAutoscaler.registered_workers
|
|
325
|
+
# => [:critical_worker, :default_worker, :batch_worker]
|
|
326
|
+
|
|
327
|
+
# Get configuration for a specific worker
|
|
328
|
+
config = SolidQueueHerokuAutoscaler.config(:critical_worker)
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
## How It Works
|
|
332
|
+
|
|
333
|
+
### Metrics Collection
|
|
334
|
+
|
|
335
|
+
The autoscaler queries Solid Queue's PostgreSQL tables to collect:
|
|
336
|
+
|
|
337
|
+
- **Queue depth**: Count of jobs in `solid_queue_ready_executions`
|
|
338
|
+
- **Oldest job age**: Time since oldest job was enqueued (latency)
|
|
339
|
+
- **Throughput**: Jobs completed in the last minute
|
|
340
|
+
- **Active workers**: Workers with recent heartbeats
|
|
341
|
+
- **Per-queue breakdown**: Job counts by queue name
|
|
342
|
+
|
|
343
|
+
When `queues` is configured, metrics are filtered to only those queues.
|
|
344
|
+
|
|
345
|
+
### Decision Logic
|
|
346
|
+
|
|
347
|
+
**Scale Up** when ANY of these conditions are met:
|
|
348
|
+
- Queue depth >= `scale_up_queue_depth`
|
|
349
|
+
- Oldest job age >= `scale_up_latency_seconds`
|
|
350
|
+
|
|
351
|
+
**Scale Down** when ALL of these conditions are met:
|
|
352
|
+
- Queue depth <= `scale_down_queue_depth`
|
|
353
|
+
- Oldest job age <= `scale_down_latency_seconds`
|
|
354
|
+
- OR queue is completely idle (no pending or claimed jobs)
|
|
355
|
+
|
|
356
|
+
**No Change** when:
|
|
357
|
+
- Already at min/max workers
|
|
358
|
+
- Within cooldown period
|
|
359
|
+
- Metrics are in normal range
|
|
360
|
+
|
|
361
|
+
### Scaling Strategies
|
|
362
|
+
|
|
363
|
+
**Fixed Strategy** (default): Adds/removes a fixed number of workers per scaling event.
|
|
364
|
+
|
|
365
|
+
```ruby
|
|
366
|
+
config.scaling_strategy = :fixed
|
|
367
|
+
config.scale_up_increment = 2 # Add 2 workers when scaling up
|
|
368
|
+
config.scale_down_decrement = 1 # Remove 1 worker when scaling down
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
**Proportional Strategy**: Scales based on how far over/under thresholds you are.
|
|
372
|
+
|
|
373
|
+
```ruby
|
|
374
|
+
config.scaling_strategy = :proportional
|
|
375
|
+
config.scale_up_jobs_per_worker = 50 # Add 1 worker per 50 jobs over threshold
|
|
376
|
+
config.scale_up_latency_per_worker = 60 # Add 1 worker per 60s over threshold
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
### Singleton Execution
|
|
380
|
+
|
|
381
|
+
PostgreSQL advisory locks ensure only one autoscaler instance runs at a time, even across multiple dynos/pods. Each worker configuration gets its own lock key, so different worker types can scale simultaneously.
|
|
382
|
+
|
|
383
|
+
### Cooldowns
|
|
384
|
+
|
|
385
|
+
After each scaling event, a cooldown period prevents additional scaling:
|
|
386
|
+
- Prevents "flapping" between states
|
|
387
|
+
- Gives the platform time to spin up new workers
|
|
388
|
+
- Allows queue to stabilize after scaling
|
|
389
|
+
|
|
390
|
+
Cooldowns are tracked per-worker type, so scaling one worker doesn't block scaling another.
|
|
391
|
+
|
|
392
|
+
## Environment Variables
|
|
393
|
+
|
|
394
|
+
### Heroku
|
|
395
|
+
|
|
396
|
+
| Variable | Description | Required |
|
|
397
|
+
|----------|-------------|----------|
|
|
398
|
+
| `HEROKU_API_KEY` | Heroku Platform API token | Yes |
|
|
399
|
+
| `HEROKU_APP_NAME` | Name of your Heroku app | Yes |
|
|
400
|
+
|
|
401
|
+
### Kubernetes
|
|
402
|
+
|
|
403
|
+
| Variable | Description | Required |
|
|
404
|
+
|----------|-------------|----------|
|
|
405
|
+
| `KUBERNETES_NAMESPACE` | Kubernetes namespace | No (defaults to 'default') |
|
|
406
|
+
|
|
407
|
+
## Dry Run Mode
|
|
408
|
+
|
|
409
|
+
Test the autoscaler without making actual changes:
|
|
410
|
+
|
|
411
|
+
```ruby
|
|
412
|
+
SolidQueueHerokuAutoscaler.configure do |config|
|
|
413
|
+
config.dry_run = true
|
|
414
|
+
# ... other config
|
|
415
|
+
end
|
|
416
|
+
```
|
|
417
|
+
|
|
418
|
+
In dry-run mode, all decisions are logged but no platform API calls are made.
|
|
419
|
+
|
|
420
|
+
## Troubleshooting
|
|
421
|
+
|
|
422
|
+
### "Could not acquire advisory lock"
|
|
423
|
+
|
|
424
|
+
Another autoscaler instance is currently running. This is expected behavior — only one instance should run at a time per worker type.
|
|
425
|
+
|
|
426
|
+
### "Cooldown active"
|
|
427
|
+
|
|
428
|
+
A recent scaling event triggered the cooldown. Wait for the cooldown to expire or adjust `cooldown_seconds`.
|
|
429
|
+
|
|
430
|
+
### Workers not scaling
|
|
431
|
+
|
|
432
|
+
1. Check that `enabled` is `true`
|
|
433
|
+
2. Verify platform credentials are set correctly
|
|
434
|
+
3. Check metrics with `rake solid_queue_autoscaler:metrics`
|
|
435
|
+
4. Enable dry-run to see what decisions would be made
|
|
436
|
+
5. Check the logs for error messages
|
|
437
|
+
|
|
438
|
+
### Kubernetes authentication issues
|
|
439
|
+
|
|
440
|
+
1. Ensure the service account has permissions to patch deployments
|
|
441
|
+
2. Check namespace is correct
|
|
442
|
+
3. Verify deployment name matches exactly
|
|
443
|
+
|
|
444
|
+
## Architecture Notes
|
|
445
|
+
|
|
446
|
+
This gem acts as a **control plane** for Solid Queue:
|
|
447
|
+
|
|
448
|
+
- **External to workers**: The autoscaler must not depend on the workers it's scaling
|
|
449
|
+
- **Singleton**: Advisory locks ensure only one instance runs globally per worker type
|
|
450
|
+
- **Dedicated queue**: Runs on its own queue to avoid competing with business jobs
|
|
451
|
+
- **Conservative**: Defaults to gradual scaling with cooldowns
|
|
452
|
+
- **Multi-tenant**: Each worker configuration is independent
|
|
453
|
+
|
|
454
|
+
## License
|
|
455
|
+
|
|
456
|
+
MIT License. See [LICENSE.txt](LICENSE.txt) for details.
|
|
457
|
+
|
|
458
|
+
## Contributing
|
|
459
|
+
|
|
460
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and contribution guidelines.
|
|
461
|
+
|
|
462
|
+
1. Fork the repository
|
|
463
|
+
2. Create your feature branch (`git checkout -b feature/my-feature`)
|
|
464
|
+
3. Write tests for your changes
|
|
465
|
+
4. Ensure all tests pass (`bundle exec rspec`)
|
|
466
|
+
5. Ensure RuboCop passes (`bundle exec rubocop`)
|
|
467
|
+
6. Submit a pull request
|
|
468
|
+
|
|
469
|
+
## Links
|
|
470
|
+
|
|
471
|
+
- [GitHub Repository](https://github.com/reillyse/solid_queue_heroku_autoscaler)
|
|
472
|
+
- [RubyGems](https://rubygems.org/gems/solid_queue_heroku_autoscaler)
|
|
473
|
+
- [Changelog](CHANGELOG.md)
|
|
474
|
+
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/generators'
|
|
4
|
+
|
|
5
|
+
module SolidQueueHerokuAutoscaler
|
|
6
|
+
module Generators
|
|
7
|
+
class InstallGenerator < Rails::Generators::Base
|
|
8
|
+
source_root File.expand_path('templates', __dir__)
|
|
9
|
+
|
|
10
|
+
desc 'Creates a SolidQueueHerokuAutoscaler initializer'
|
|
11
|
+
|
|
12
|
+
def copy_initializer
|
|
13
|
+
template 'initializer.rb', 'config/initializers/solid_queue_autoscaler.rb'
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def show_readme
|
|
17
|
+
readme 'README' if behavior == :invoke
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rails/generators'
|
|
4
|
+
require 'rails/generators/active_record'
|
|
5
|
+
|
|
6
|
+
module SolidQueueHerokuAutoscaler
|
|
7
|
+
module Generators
|
|
8
|
+
class MigrationGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path('templates', __dir__)
|
|
12
|
+
|
|
13
|
+
desc 'Creates the migration for SolidQueueHerokuAutoscaler state table'
|
|
14
|
+
|
|
15
|
+
def create_migration_file
|
|
16
|
+
migration_template 'create_solid_queue_autoscaler_state.rb.erb',
|
|
17
|
+
'db/migrate/create_solid_queue_autoscaler_state.rb'
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def migration_version
|
|
23
|
+
return unless defined?(ActiveRecord::VERSION)
|
|
24
|
+
|
|
25
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|