solid_queue_autoscaler 1.0.7 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c25a5d2a7570ea01da30c54a42624674b23d7112bef33da9d82c5ec1257933a5
-  data.tar.gz: '081a9f3000bcb460bc7254e7597c2f3e0ddd7afd9517f1f213bf1bd3cc2020c7'
+  metadata.gz: 2caadbfc7fd3df7b06be9dbea2e3e667b5e8b49e8c302720fb07d56cfb1a7d1c
+  data.tar.gz: 0b4c14b56f29e1ed6537ecac97d279661d08d943320df901bf7c73f3a6bd8694
 SHA512:
-  metadata.gz: 580ac675beb8175d1c4897bbb251d20ab7475779a8d49d2fabfd2ea800f8f27b079df5c7f627a27e22ee2220a6e8d5fa06411a00c846888c5c456a32cc3bbdc6
-  data.tar.gz: b1a8f294a803035338f49436f98d479ee8ed39a2f5bdbb7b64e0701a5e00365a05d32e7e7ba8385a89d9dd72ba2c1f09019e9d7ce79e5de3f1e0a2a20ff50180
+  metadata.gz: f2b316153846191155d72961c4be0e95b6d6c2dd71aedc32a268dff92bd570736e703cc2a8e1e7be564b9899d407c80959350014a3ea167f0f3472d0224d2109
+  data.tar.gz: 4048d221c232b9b079d08f24e571dcc9b56c7c4b3c69d7ce14262e5986568c87c93d07152664667021e981018da79ac581335db82420070c5fcc243bfe626efd

data/CHANGELOG.md

@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.0.8] - 2025-01-17
+
+### Added
+- **`job_queue` configuration option** - Configure which queue the AutoscaleJob runs on (default: `:autoscaler`)
+- **`job_priority` configuration option** - Set job priority for AutoscaleJob (lower = higher priority)
+- **Multi-database migration support** - Migration templates now automatically create tables in the same database as Solid Queue
+- **Common Configuration Examples** - New README section with 8 copy-paste ready configurations for different use cases:
+  - Simple/Starter setup
+  - Cost-Optimized (scale to zero)
+  - E-Commerce/SaaS (multiple worker types)
+  - High-Volume API (webhook processing with proportional scaling)
+  - Data Processing/ETL
+  - High-Availability
+  - Kubernetes
+  - Development/Testing
+- **Expanded Troubleshooting** - 15+ new troubleshooting topics with code examples
+- **Configuration Comparison Table** - Quick reference for common configurations
+
+### Changed
+- AutoscaleJob now uses `queue_as` and `queue_with_priority` blocks for dynamic queue/priority selection
+- Each worker configuration can have its own `job_queue` and `job_priority` settings
+
 ## [1.0.7] - 2025-01-16
 
 ### Fixed

data/README.md

@@ -240,6 +240,13 @@ Scaling down triggers when **ALL** thresholds are met:
 | `scale_down_cooldown_seconds` | Integer | `nil` | Override for scale-down cooldown |
 | `persist_cooldowns` | Boolean | `true` | Save cooldowns to database |
 
+### AutoscaleJob Settings
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `job_queue` | Symbol/String | `:autoscaler` | Queue for the AutoscaleJob |
+| `job_priority` | Integer | `nil` | Priority for the AutoscaleJob (lower = higher priority) |
+
 ### Heroku-Specific
 
 | Option | Type | Default | Description |
@@ -256,6 +263,482 @@ Scaling down triggers when **ALL** thresholds are met:
 | `kubernetes_deployment` | String | `nil` | Deployment name to scale |
 | `kubernetes_config_path` | String | `nil` | Path to kubeconfig (optional) |
 
+## Common Configuration Examples
+
+These examples show typical setups for different use cases. Copy and adapt them to your needs.
+
+### Simple Setup (Single Worker, Heroku)
+
+Ideal for small apps, side projects, or getting started:
+
+```ruby
+# config/initializers/solid_queue_autoscaler.rb
+SolidQueueAutoscaler.configure do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'worker'
+
+  config.min_workers = 1
+  config.max_workers = 5
+
+  # Scale up when queue backs up
+  config.scale_up_queue_depth = 50
+  config.scale_up_latency_seconds = 180  # 3 minutes
+
+  # Scale down when queue is nearly empty
+  config.scale_down_queue_depth = 5
+  config.scale_down_latency_seconds = 30
+
+  # Safety: only run in production
+  config.dry_run = !Rails.env.production?
+  config.enabled = Rails.env.production?
+end
+```
+
+```yaml
+# config/recurring.yml
+autoscaler:
+  class: SolidQueueAutoscaler::AutoscaleJob
+  queue: autoscaler
+  schedule: every 30 seconds
+```
+
+---
+
+### Cost-Optimized Setup (Scale to Zero)
+
+For apps with sporadic workloads where you want to minimize costs during idle periods:
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'worker'
+
+  # Allow scaling to zero - no workers when idle
+  config.min_workers = 0
+  config.max_workers = 5
+
+  # Scale up immediately when any job is queued
+  config.scale_up_queue_depth = 1
+  config.scale_up_latency_seconds = 60  # 1 minute
+
+  # Scale down aggressively when empty
+  config.scale_down_queue_depth = 0
+  config.scale_down_latency_seconds = 10
+
+  # Shorter cooldowns for faster response
+  config.scale_up_cooldown_seconds = 30
+  config.scale_down_cooldown_seconds = 300  # 5 min before scaling to zero
+
+  config.enabled = Rails.env.production?
+end
+```
+
+**⚠️ Note:** With `min_workers = 0`, there's cold-start latency when the first job arrives. The autoscaler must run on a web dyno or separate process, not on the workers themselves.
+
+---
+
+### E-Commerce / SaaS (Multiple Worker Types)
+
+For apps with different job priorities (payments, notifications, reports):
+
+```ruby
+# Critical jobs - payments, webhooks, user-facing notifications
+SolidQueueAutoscaler.configure(:critical_worker) do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'critical_worker'
+  
+  config.queues = ['critical', 'payments', 'webhooks']
+  
+  # Always have capacity, scale aggressively
+  config.min_workers = 2
+  config.max_workers = 10
+  config.scale_up_queue_depth = 5
+  config.scale_up_latency_seconds = 30
+  
+  # Short cooldowns for responsiveness
+  config.cooldown_seconds = 60
+  
+  # High-priority autoscaler job
+  config.job_queue = :autoscaler
+  config.job_priority = 0
+end
+
+# Default jobs - emails, notifications, analytics
+SolidQueueAutoscaler.configure(:default_worker) do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'worker'
+  
+  config.queues = ['default', 'mailers', 'analytics']
+  
+  # Standard capacity, moderate scaling
+  config.min_workers = 1
+  config.max_workers = 8
+  config.scale_up_queue_depth = 100
+  config.scale_up_latency_seconds = 300
+  
+  config.cooldown_seconds = 120
+end
+
+# Batch jobs - reports, exports, data processing
+SolidQueueAutoscaler.configure(:batch_worker) do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'batch_worker'
+  
+  config.queues = ['batch', 'reports', 'exports']
+  
+  # Scale to zero when no batch jobs, scale up for any batch work
+  config.min_workers = 0
+  config.max_workers = 3
+  config.scale_up_queue_depth = 1
+  config.scale_down_queue_depth = 0
+  
+  # Long cooldowns - batch jobs take time
+  config.cooldown_seconds = 300
+end
+```
+
+```yaml
+# config/recurring.yml
+# Scale critical workers frequently
+autoscaler_critical:
+  class: SolidQueueAutoscaler::AutoscaleJob
+  queue: autoscaler
+  schedule: every 15 seconds
+  args: [:critical_worker]
+
+# Scale default workers normally
+autoscaler_default:
+  class: SolidQueueAutoscaler::AutoscaleJob
+  queue: autoscaler
+  schedule: every 30 seconds
+  args: [:default_worker]
+
+# Scale batch workers less frequently
+autoscaler_batch:
+  class: SolidQueueAutoscaler::AutoscaleJob
+  queue: autoscaler
+  schedule: every 60 seconds
+  args: [:batch_worker]
+```
+
+---
+
+### High-Volume API (Webhook Processing)
+
+For apps processing many incoming webhooks or API callbacks:
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'worker'
+
+  config.queues = ['webhooks', 'callbacks', 'api_jobs']
+
+  # Maintain baseline capacity
+  config.min_workers = 2
+  config.max_workers = 20
+
+  # Proportional scaling - scale based on actual load
+  config.scaling_strategy = :proportional
+  config.scale_up_queue_depth = 50
+  config.scale_up_latency_seconds = 60
+  
+  # Add 1 worker per 25 jobs over threshold
+  config.scale_up_jobs_per_worker = 25
+  # Add 1 worker per 30 seconds over latency threshold
+  config.scale_up_latency_per_worker = 30
+  
+  # Scale down when under capacity
+  config.scale_down_queue_depth = 10
+  config.scale_down_jobs_per_worker = 50
+
+  # Fast cooldowns for responsive scaling
+  config.scale_up_cooldown_seconds = 30
+  config.scale_down_cooldown_seconds = 120
+  
+  config.job_priority = 0  # Process autoscaler jobs first
+end
+```
+
+---
+
+### Data Processing / ETL Pipeline
+
+For apps with heavy data processing, imports, or batch ETL jobs:
+
+```ruby
+SolidQueueAutoscaler.configure(:etl_worker) do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'etl_worker'
+
+  config.queues = ['imports', 'exports', 'etl', 'data_sync']
+
+  # Scale to zero when no work, burst when needed
+  config.min_workers = 0
+  config.max_workers = 10
+
+  # Scale up as soon as work is queued
+  config.scale_up_queue_depth = 1
+  config.scale_up_latency_seconds = 120
+  
+  # Use fixed scaling for predictable behavior
+  config.scaling_strategy = :fixed
+  config.scale_up_increment = 2  # Add 2 workers at a time
+  config.scale_down_decrement = 1
+
+  # Long cooldowns - ETL jobs are long-running
+  config.scale_up_cooldown_seconds = 120
+  config.scale_down_cooldown_seconds = 600  # 10 minutes
+
+  # Scale down only when truly idle
+  config.scale_down_queue_depth = 0
+  config.scale_down_latency_seconds = 0
+end
+```
+
+---
+
+### High-Availability Setup
+
+For mission-critical apps requiring guaranteed capacity:
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'worker'
+
+  # Always maintain minimum capacity
+  config.min_workers = 3
+  config.max_workers = 15
+
+  # Scale up proactively before queue backs up
+  config.scale_up_queue_depth = 25
+  config.scale_up_latency_seconds = 60
+
+  # Conservative scale-down
+  config.scale_down_queue_depth = 5
+  config.scale_down_latency_seconds = 15
+
+  # Longer cooldowns to prevent flapping
+  config.cooldown_seconds = 180
+  config.scale_down_cooldown_seconds = 300  # Extra cautious on scale-down
+
+  # Record all events for monitoring
+  config.record_events = true
+  config.record_all_events = true  # Even no-change events
+end
+```
+
+---
+
+### Kubernetes Setup
+
+For apps deployed on Kubernetes:
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  config.adapter = :kubernetes
+  config.kubernetes_namespace = ENV.fetch('K8S_NAMESPACE', 'production')
+  config.kubernetes_deployment = 'solid-queue-worker'
+  
+  # Optional: specify kubeconfig for local development
+  # config.kubernetes_kubeconfig = '~/.kube/config'
+  # config.kubernetes_context = 'my-cluster'
+
+  config.min_workers = 2  # Minimum replicas
+  config.max_workers = 20
+
+  config.scale_up_queue_depth = 100
+  config.scale_up_latency_seconds = 180
+  
+  config.scale_down_queue_depth = 10
+  config.scale_down_latency_seconds = 30
+
+  # K8s scaling can be faster than Heroku
+  config.cooldown_seconds = 60
+  
+  config.enabled = Rails.env.production?
+end
+```
+
+**Required RBAC configuration:**
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: solid-queue-autoscaler
+  namespace: production
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments", "deployments/scale"]
+  verbs: ["get", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: solid-queue-autoscaler
+  namespace: production
+subjects:
+- kind: ServiceAccount
+  name: solid-queue-autoscaler
+  namespace: production
+roleRef:
+  kind: Role
+  name: solid-queue-autoscaler
+  apiGroup: rbac.authorization.k8s.io
+```
+
+---
+
+### Development / Testing Setup
+
+For local development and CI environments:
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  config.adapter = :heroku
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'worker'
+
+  config.min_workers = 1
+  config.max_workers = 3
+
+  config.scale_up_queue_depth = 10
+  config.scale_up_latency_seconds = 60
+
+  # IMPORTANT: Disable in development, use dry_run in staging
+  case Rails.env
+  when 'production'
+    config.enabled = true
+    config.dry_run = false
+  when 'staging'
+    config.enabled = true
+    config.dry_run = true  # Log decisions but don't scale
+  else
+    config.enabled = false
+  end
+
+  # Verbose logging for debugging
+  config.logger = Logger.new(STDOUT)
+  config.logger.level = Rails.env.production? ? Logger::INFO : Logger::DEBUG
+end
+```
+
+---
+
+### Configuration Comparison
+
+| Use Case | min | max | scale_up_depth | scale_up_latency | cooldown | strategy |
+|----------|-----|-----|----------------|------------------|----------|----------|
+| Simple/Starter | 1 | 5 | 50 | 180s | 120s | fixed |
+| Cost-Optimized | 0 | 5 | 1 | 60s | 30s/300s | fixed |
+| E-Commerce Critical | 2 | 10 | 5 | 30s | 60s | fixed |
+| E-Commerce Default | 1 | 8 | 100 | 300s | 120s | fixed |
+| Webhook Processing | 2 | 20 | 50 | 60s | 30s/120s | proportional |
+| ETL/Batch | 0 | 10 | 1 | 120s | 120s/600s | fixed |
+| High-Availability | 3 | 15 | 25 | 60s | 180s/300s | fixed |
+
+## Configuring a High-Priority Queue for the Autoscaler
+
+The autoscaler job should run reliably and quickly, even when your queues are backed up. By default, the autoscaler job runs on the `:autoscaler` queue. You can configure this and set up Solid Queue to prioritize it.
+
+### Configure the Job Queue and Priority
+
+In your initializer, set the queue and priority for the autoscaler job:
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  # Use a dedicated high-priority queue for the autoscaler
+  config.job_queue = :autoscaler  # Default value
+  
+  # Or use an existing high-priority queue
+  config.job_queue = :critical
+  
+  # Set job priority (lower = higher priority, processed first)
+  # This works with queue backends that support job-level priority like Solid Queue
+  config.job_priority = 0  # Highest priority
+  
+  # ... other config
+end
+```
+
+For multi-worker configurations, each worker type can have its own queue and priority:
+
+```ruby
+SolidQueueAutoscaler.configure(:critical_worker) do |config|
+  config.job_queue = :autoscaler_critical
+  config.job_priority = 0  # Highest priority for critical worker scaling
+  # ... other config
+end
+
+SolidQueueAutoscaler.configure(:default_worker) do |config|
+  config.job_queue = :autoscaler_default
+  config.job_priority = 10  # Lower priority for default worker scaling
+  # ... other config
+end
+```
+
+### Configure Solid Queue to Prioritize the Autoscaler
+
+In your `config/solid_queue.yml`, ensure the autoscaler queue is processed by a dedicated worker or listed first in the queue order:
+
+```yaml
+# Option 1: Dedicated dispatcher/worker for autoscaler (recommended)
+production:
+  dispatchers:
+    - polling_interval: 1
+      batch_size: 500
+      concurrency_maintenance_interval: 30
+
+  workers:
+    # Dedicated worker for autoscaler - always responsive
+    - queues: [autoscaler]
+      threads: 1
+      processes: 1
+      polling_interval: 0.5  # Check frequently
+    
+    # Main workers for business logic
+    - queues: [critical, default, mailers]
+      threads: 5
+      processes: 2
+      polling_interval: 1
+```
+
+```yaml
+# Option 2: Include autoscaler first in queue list (simpler)
+production:
+  workers:
+    - queues: [autoscaler, critical, default, mailers]
+      threads: 5
+      processes: 2
+```
+
+Solid Queue processes queues in order, so listing `autoscaler` first ensures those jobs are picked up before others.
+
+### Why This Matters
+
+- **Responsiveness**: When your queues are backed up, you want the autoscaler to scale up workers quickly
+- **Reliability**: A dedicated queue prevents autoscaler jobs from waiting behind thousands of business jobs
+- **Isolation**: Separating autoscaler jobs makes monitoring and debugging easier
+
 ## Usage
 
 ### Running as a Solid Queue Recurring Job (Recommended)
@@ -506,10 +989,32 @@ KEEP_DAYS=7 bundle exec rake solid_queue_autoscaler:cleanup_events
 
 Another autoscaler instance is currently running. This is expected behavior — only one instance should run at a time per worker type.
 
+**If you believe no other instance is running:**
+
+```ruby
+# Check for stale advisory locks
+ActiveRecord::Base.connection.execute(<<~SQL)
+  SELECT * FROM pg_locks WHERE locktype = 'advisory'
+SQL
+
+# Force release a stuck lock (use with caution!)
+lock_key = SolidQueueAutoscaler.config.lock_key
+lock_id = Zlib.crc32(lock_key) & 0x7FFFFFFF
+ActiveRecord::Base.connection.execute("SELECT pg_advisory_unlock(#{lock_id})")
+```
+
 ### "Cooldown active"
 
 A recent scaling event triggered the cooldown. Wait for the cooldown to expire or adjust `cooldown_seconds`.
 
+```ruby
+# Check cooldown status
+bundle exec rake solid_queue_autoscaler:cooldown
+
+# Reset cooldowns (for testing only)
+SolidQueueAutoscaler::CooldownTracker.reset!
+```
+
 ### Workers not scaling
 
 1. Check that `enabled` is `true`
@@ -518,12 +1023,263 @@ A recent scaling event triggered the cooldown. Wait for the cooldown to expire o
 4. Enable dry-run to see what decisions would be made
 5. Check the logs for error messages
 
+**Debug with a manual scale attempt:**
+
+```ruby
+# Check configuration
+config = SolidQueueAutoscaler.config
+puts "Enabled: #{config.enabled?}"
+puts "Dry Run: #{config.dry_run?}"
+puts "API Key Set: #{config.heroku_api_key.present?}"
+
+# Check current metrics
+metrics = SolidQueueAutoscaler.metrics
+puts "Queue depth: #{metrics.queue_depth}"
+puts "Latency: #{metrics.oldest_job_age_seconds}s"
+
+# Try a manual scale
+result = SolidQueueAutoscaler.scale!
+puts result.decision.inspect if result.decision
+puts result.skipped_reason if result.skipped?
+puts result.error if result.error
+```
+
+### Workers not scaling down
+
+Scale-down requires **ALL** conditions to be met:
+
+```ruby
+metrics = SolidQueueAutoscaler.metrics
+config = SolidQueueAutoscaler.config
+
+puts "Queue depth: #{metrics.queue_depth} (threshold: <= #{config.scale_down_queue_depth})"
+puts "Latency: #{metrics.oldest_job_age_seconds}s (threshold: <= #{config.scale_down_latency_seconds}s)"
+puts "Claimed jobs: #{metrics.claimed_jobs}"  # Must be 0 for idle scale-down
+puts "Current workers: #{SolidQueueAutoscaler.current_workers}"
+puts "Min workers: #{config.min_workers}"  # Can't scale below this
+```
+
+### Heroku API errors
+
+**401 Unauthorized:**
+
+```bash
+# Check if API key is valid
+heroku authorizations
+
+# Create a new authorization
+heroku authorizations:create -d "Solid Queue Autoscaler"
+```
+
+**404 Not Found:**
+
+```bash
+# Verify app name
+heroku apps
+heroku apps:info -a $HEROKU_APP_NAME
+```
+
+**429 Rate Limited:**
+
+Increase cooldown to reduce API calls:
+
+```ruby
+config.cooldown_seconds = 180  # 3 minutes instead of default 2
+```
+
 ### Kubernetes authentication issues
 
 1. Ensure the service account has permissions to patch deployments
 2. Check namespace is correct
 3. Verify deployment name matches exactly
 
+**Check RBAC permissions:**
+
+```yaml
+# Required RBAC rules for the autoscaler service account
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: solid-queue-autoscaler
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments", "deployments/scale"]
+  verbs: ["get", "patch", "update"]
+```
+
+### AutoscaleJob not running
+
+**Check recurring.yml configuration:**
+
+```yaml
+# config/recurring.yml
+autoscaler:
+  class: SolidQueueAutoscaler::AutoscaleJob
+  queue: autoscaler
+  schedule: every 30 seconds
+```
+
+**Ensure a worker processes the autoscaler queue:**
+
+```yaml
+# config/solid_queue.yml
+workers:
+  - queues: [autoscaler]  # Must include autoscaler queue
+    threads: 1
+```
+
+**Test manual enqueue:**
+
+```ruby
+SolidQueueAutoscaler::AutoscaleJob.perform_later
+```
+
+### Multi-worker configuration issues
+
+**"Unknown worker: :my_worker":**
+
+Ensure you've configured the worker before referencing it:
+
+```ruby
+# Configure the worker first
+SolidQueueAutoscaler.configure(:my_worker) do |config|
+  config.heroku_api_key = ENV['HEROKU_API_KEY']
+  config.heroku_app_name = ENV['HEROKU_APP_NAME']
+  config.process_type = 'my_worker'
+end
+
+# Then reference it
+SolidQueueAutoscaler.scale!(:my_worker)
+```
+
+**List all registered workers:**
+
+```ruby
+SolidQueueAutoscaler.registered_workers
+# => [:default, :critical_worker, :batch_worker]
+```
+
+### Database/Migration issues
+
+**"relation 'solid_queue_autoscaler_state' does not exist":**
+
+```bash
+rails generate solid_queue_autoscaler:migration
+rails db:migrate
+```
+
+**"relation 'solid_queue_ready_executions' does not exist":**
+
+Solid Queue tables are missing. Run Solid Queue migrations:
+
+```bash
+rails solid_queue:install:migrations
+rails db:migrate
+```
+
+**Multi-database setup (Solid Queue on separate database):**
+
+The autoscaler automatically detects `SolidQueue::Record.connection`. If auto-detection fails:
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  config.database_connection = SolidQueue::Record.connection
+end
+```
+
+### Dashboard not loading
+
+**404 when visiting /autoscaler:**
+
+Ensure the engine is mounted in `config/routes.rb`:
+
+```ruby
+mount SolidQueueAutoscaler::Dashboard::Engine => "/autoscaler"
+```
+
+**"ActionView::MissingTemplate" errors:**
+
+Run the dashboard generator:
+
+```bash
+rails generate solid_queue_autoscaler:dashboard
+rails db:migrate
+```
+
+### Wrong process type being scaled
+
+```ruby
+# Check what process type is configured
+puts SolidQueueAutoscaler.config.process_type
+
+# Verify it matches your Procfile
+# Procfile:
+# web: bundle exec puma -C config/puma.rb
+# worker: bundle exec rake solid_queue:start  # <- This is "worker"
+```
+
+### Scaling too aggressively or too slowly
+
+**Scaling up too often (flapping):**
+
+```ruby
+config.cooldown_seconds = 180           # Increase cooldown
+config.scale_up_cooldown_seconds = 120  # Or set scale-up specific cooldown
+config.scale_up_queue_depth = 200       # Increase threshold
+```
+
+**Not scaling up fast enough:**
+
+```ruby
+config.scale_up_queue_depth = 50        # Lower threshold
+config.scale_up_latency_seconds = 120   # Trigger on 2 min latency
+config.cooldown_seconds = 60            # Reduce cooldown
+config.scaling_strategy = :proportional # Scale based on load, not fixed increment
+config.scale_up_jobs_per_worker = 25    # More workers per jobs over threshold
+```
+
+**Not scaling down:**
+
+```ruby
+config.scale_down_queue_depth = 5       # More aggressive scale-down threshold  
+config.scale_down_latency_seconds = 10  # Tighter latency requirement
+config.min_workers = 0                  # Allow scaling to zero (if appropriate)
+```
+
+### Debugging tips
+
+**Enable debug logging:**
+
+```ruby
+SolidQueueAutoscaler.configure do |config|
+  config.logger = Logger.new(STDOUT)
+  config.logger.level = Logger::DEBUG
+end
+```
+
+**Simulate a scaling decision without making changes:**
+
+```ruby
+metrics = SolidQueueAutoscaler.metrics
+workers = SolidQueueAutoscaler.current_workers
+engine = SolidQueueAutoscaler::DecisionEngine.new(config: SolidQueueAutoscaler.config)
+decision = engine.decide(metrics: metrics, current_workers: workers)
+
+puts "Action: #{decision.action}"    # :scale_up, :scale_down, or :no_change
+puts "From: #{decision.from} -> To: #{decision.to}"
+puts "Reason: #{decision.reason}"
+```
+
+**Run diagnostics:**
+
+```bash
+bundle exec rake solid_queue_autoscaler:metrics
+bundle exec rake solid_queue_autoscaler:formation
+bundle exec rake solid_queue_autoscaler:cooldown
+```
+
+For more detailed troubleshooting, see [docs/troubleshooting.md](docs/troubleshooting.md).
+
 ## Architecture Notes
 
 This gem acts as a **control plane** for Solid Queue:

data/lib/generators/solid_queue_autoscaler/templates/create_solid_queue_autoscaler_events.rb.erb

@@ -1,6 +1,15 @@
 # frozen_string_literal: true
 
 class CreateSolidQueueAutoscalerEvents < ActiveRecord::Migration<%= migration_version %>
+  # Use the same database connection as SolidQueue for multi-database setups
+  def self.connection
+    if defined?(SolidQueue::Record) && SolidQueue::Record.respond_to?(:connection)
+      SolidQueue::Record.connection
+    else
+      super
+    end
+  end
+
   def change
     create_table :solid_queue_autoscaler_events do |t|
       t.string :worker_name, null: false

data/lib/generators/solid_queue_autoscaler/templates/create_solid_queue_autoscaler_state.rb.erb

@@ -1,6 +1,15 @@
 # frozen_string_literal: true
 
 class CreateSolidQueueAutoscalerState < ActiveRecord::Migration<%= migration_version %>
+  # Use the same database connection as SolidQueue for multi-database setups
+  def self.connection
+    if defined?(SolidQueue::Record) && SolidQueue::Record.respond_to?(:connection)
+      SolidQueue::Record.connection
+    else
+      super
+    end
+  end
+
   def change
     create_table :solid_queue_autoscaler_state do |t|
       t.string :key, null: false

data/lib/generators/solid_queue_autoscaler/templates/initializer.rb

@@ -55,4 +55,10 @@ SolidQueueAutoscaler.configure do |config|
   config.record_events = true
   # Also record no_change events (verbose, generates many records)
   # config.record_all_events = false
+
+  # AutoscaleJob Settings
+  # Queue for the autoscaler job (use a fast/high-priority Solid Queue queue)
+  config.job_queue = :autoscaler
+  # Priority for the autoscaler job (lower = higher priority, nil = default)
+  # config.job_priority = 0  # Uncomment to set highest priority
 end

data/lib/solid_queue_autoscaler/autoscale_job.rb

@@ -2,7 +2,40 @@
 
 module SolidQueueAutoscaler
   class AutoscaleJob < ActiveJob::Base
-    queue_as :autoscaler
+    # Use configured queue for the target worker (defaults to :autoscaler)
+    queue_as do
+      # perform(worker_name = :default)
+      worker_name = arguments.first
+
+      # When scaling all workers, or when worker_name is nil, use the default configuration
+      config_name =
+        if worker_name.nil? || worker_name == :all || worker_name == "all"
+          :default
+        else
+          # Handle both Symbol and String values safely
+          worker_name.to_sym rescue :default
+        end
+
+      SolidQueueAutoscaler.config(config_name).job_queue || :autoscaler
+    end
+
+    # Use configured priority for the target worker (defaults to nil/no priority)
+    queue_with_priority do
+      # perform(worker_name = :default)
+      worker_name = arguments.first
+
+      # When scaling all workers, or when worker_name is nil, use the default configuration
+      config_name =
+        if worker_name.nil? || worker_name == :all || worker_name == "all"
+          :default
+        else
+          # Handle both Symbol and String values safely
+          worker_name.to_sym rescue :default
+        end
+
+      SolidQueueAutoscaler.config(config_name).job_priority
+    end
+
     discard_on ConfigurationError
 
     # Scale a specific worker type, or all workers if :all is passed

data/lib/solid_queue_autoscaler/configuration.rb

@@ -63,6 +63,9 @@ module SolidQueueAutoscaler
     # Dashboard/event recording settings
     attr_accessor :record_events, :record_all_events
 
+    # AutoscaleJob settings
+    attr_accessor :job_queue, :job_priority
+
     def initialize
       # Configuration name (auto-set when using named configurations)
       @name = :default
@@ -128,6 +131,10 @@ module SolidQueueAutoscaler
       # Dashboard/event recording settings
       @record_events = true # Record scale events to database
       @record_all_events = false # Also record no_change events (verbose)
+
+      # AutoscaleJob settings
+      @job_queue = :autoscaler # Queue name for the autoscaler job
+      @job_priority = nil # Job priority (lower = higher priority, nil = default)
     end
 
     # Returns the lock key, auto-generating based on name if not explicitly set

data/lib/solid_queue_autoscaler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SolidQueueAutoscaler
-  VERSION = '1.0.7'
+  VERSION = '1.0.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue_autoscaler
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.8
 platform: ruby
 authors:
 - reillyse
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement