solid_queue_autoscaler 1.0.7 → 1.0.9

data/lib/solid_queue_autoscaler/scaler.rb

@@ -82,6 +82,7 @@ module SolidQueueAutoscaler
       @metrics_collector = Metrics.new(config: @config)
       @decision_engine = DecisionEngine.new(config: @config)
       @adapter = @config.adapter
+      @cooldown_tracker = nil # Lazy-loaded when persist_cooldowns is enabled
     end
 
     def run
@@ -124,44 +125,117 @@ module SolidQueueAutoscaler
     end
 
     def apply_decision(decision, metrics)
-      @adapter.scale(decision.to)
+      # Re-verify current workers to catch race conditions where another instance
+      # may have scaled while we were making our decision
+      verified_current = @adapter.current_workers
+      
+      if verified_current != decision.from
+        logger.warn(
+          "[Autoscaler] Worker count changed during decision: expected=#{decision.from}, actual=#{verified_current}. " \
+          "Re-evaluating..."
+        )
+        
+        # If we're already at or above max, don't scale up
+        if decision.scale_up? && verified_current >= @config.max_workers
+          return skipped_result(
+            "Aborted scale_up: already at max_workers (#{verified_current} >= #{@config.max_workers})",
+            decision: decision,
+            metrics: metrics
+          )
+        end
+        
+        # If we're already at or below min, don't scale down
+        if decision.scale_down? && verified_current <= @config.min_workers
+          return skipped_result(
+            "Aborted scale_down: already at min_workers (#{verified_current} <= #{@config.min_workers})",
+            decision: decision,
+            metrics: metrics
+          )
+        end
+      end
+
+      # Final safety clamp: never exceed configured limits
+      target = decision.to.clamp(@config.min_workers, @config.max_workers)
+      
+      if target != decision.to
+        logger.warn(
+          "[Autoscaler] Clamping target from #{decision.to} to #{target} " \
+          "(limits: #{@config.min_workers}-#{@config.max_workers})"
+        )
+        # Ensure decision reflects the clamped target for logging and events
+        decision.to = target
+      end
+      
+      @adapter.scale(target)
       record_scale_time(decision)
       record_scale_event(decision, metrics)
-
+      
       log_scale_action(decision)
 
       success_result(decision, metrics)
     end
 
     def cooldown_active?(decision)
-      config_name = @config.name
-      if decision.scale_up?
-        last_scale_up = self.class.last_scale_up_at(config_name)
-        return false unless last_scale_up
-
-        Time.current - last_scale_up < @config.effective_scale_up_cooldown
-      elsif decision.scale_down?
-        last_scale_down = self.class.last_scale_down_at(config_name)
-        return false unless last_scale_down
-
-        Time.current - last_scale_down < @config.effective_scale_down_cooldown
+      if @config.persist_cooldowns && cooldown_tracker.table_exists?
+        # Use database-persisted cooldowns (survives process restarts)
+        if decision.scale_up?
+          cooldown_tracker.cooldown_active_for_scale_up?
+        elsif decision.scale_down?
+          cooldown_tracker.cooldown_active_for_scale_down?
+        else
+          false
+        end
       else
-        false
+        # Fall back to in-memory cooldowns
+        config_name = @config.name
+        if decision.scale_up?
+          last_scale_up = self.class.last_scale_up_at(config_name)
+          return false unless last_scale_up
+
+          Time.current - last_scale_up < @config.effective_scale_up_cooldown
+        elsif decision.scale_down?
+          last_scale_down = self.class.last_scale_down_at(config_name)
+          return false unless last_scale_down
+
+          Time.current - last_scale_down < @config.effective_scale_down_cooldown
+        else
+          false
+        end
       end
     end
 
     def cooldown_remaining(decision)
-      config_name = @config.name
-      if decision.scale_up?
-        elapsed = Time.current - self.class.last_scale_up_at(config_name)
-        @config.effective_scale_up_cooldown - elapsed
+      if @config.persist_cooldowns && cooldown_tracker.table_exists?
+        # Use database-persisted cooldowns
+        if decision.scale_up?
+          cooldown_tracker.scale_up_cooldown_remaining
+        else
+          cooldown_tracker.scale_down_cooldown_remaining
+        end
       else
-        elapsed = Time.current - self.class.last_scale_down_at(config_name)
-        @config.effective_scale_down_cooldown - elapsed
+        # Fall back to in-memory cooldowns
+        config_name = @config.name
+        if decision.scale_up?
+          elapsed = Time.current - self.class.last_scale_up_at(config_name)
+          @config.effective_scale_up_cooldown - elapsed
+        else
+          elapsed = Time.current - self.class.last_scale_down_at(config_name)
+          @config.effective_scale_down_cooldown - elapsed
+        end
       end
     end
 
     def record_scale_time(decision)
+      if @config.persist_cooldowns && cooldown_tracker.table_exists?
+        # Use database-persisted cooldowns
+        if decision.scale_up?
+          cooldown_tracker.record_scale_up!
+        elsif decision.scale_down?
+          cooldown_tracker.record_scale_down!
+        end
+      end
+
+      # Always update in-memory cooldowns as well (for immediate effect within same process)
       config_name = @config.name
       if decision.scale_up?
         self.class.set_last_scale_up_at(config_name, Time.current)
@@ -170,6 +244,10 @@ module SolidQueueAutoscaler
       end
     end
 
+    def cooldown_tracker
+      @cooldown_tracker ||= CooldownTracker.new(config: @config, key: @config.name.to_s)
+    end
+
     def log_decision(decision, metrics)
       worker_label = @config.name == :default ? '' : "[#{@config.name}] "
       logger.info(

data/lib/solid_queue_autoscaler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SolidQueueAutoscaler
-  VERSION = '1.0.7'
+  VERSION = '1.0.9'
 end

data/lib/solid_queue_autoscaler.rb

@@ -99,6 +99,260 @@ module SolidQueueAutoscaler
         configurations[:default] = config_obj
       end
     end
+
+    # Verify the installation is complete and working.
+    # Prints a human-friendly report (when verbose: true) and returns a VerificationResult.
+    #
+    # Usage (Rails/Heroku console):
+    #   SolidQueueAutoscaler.verify_setup!
+    #   # or alias:
+    #   SolidQueueAutoscaler.verify_install!
+    #
+    # You can also inspect the returned struct:
+    #   result = SolidQueueAutoscaler.verify_setup!(verbose: false)
+    #   result.ok?   # => true/false
+    #   result.to_h  # => hash of details
+    def verify_setup!(name = :default, verbose: true)
+      result = VerificationResult.new
+      cfg = config(name)
+      connection = cfg.connection
+
+      output = []
+      output << '=' * 60
+      output << 'SolidQueueAutoscaler Setup Verification'
+      output << '=' * 60
+      output << ''
+      output << "Version: #{VERSION}"
+      output << "Configuration: #{name}"
+
+      # Check connection type (handles SolidQueue in its own DB)
+      if defined?(SolidQueue::Record) && SolidQueue::Record.respond_to?(:connection)
+        output << '✓ Using SolidQueue::Record connection (multi-database setup)'
+        result.connection_type = :solid_queue_record
+      else
+        output << '✓ Using ActiveRecord::Base connection'
+        result.connection_type = :active_record_base
+      end
+
+      # 1. Cooldown state table
+      output << ''
+      output << '-' * 60
+      output << '1. COOLDOWN STATE TABLE (solid_queue_autoscaler_state)'
+      output << '-' * 60
+
+      if connection.table_exists?(:solid_queue_autoscaler_state)
+        result.state_table_exists = true
+        output << '✓ Table exists'
+
+        columns = connection.columns(:solid_queue_autoscaler_state).map(&:name)
+        expected = %w[id key last_scale_up_at last_scale_down_at created_at updated_at]
+        missing = expected - columns
+
+        if missing.empty?
+          result.state_table_columns_ok = true
+          output << '  ✓ All expected columns present'
+        else
+          result.state_table_columns_ok = false
+          result.add_warning("State table missing columns: #{missing.join(', ')}")
+          output << "  ⚠ Missing columns: #{missing.join(', ')}"
+        end
+
+        state_count = connection.select_value('SELECT COUNT(*) FROM solid_queue_autoscaler_state').to_i
+        output << "  Current records: #{state_count}"
+      else
+        result.state_table_exists = false
+        result.add_error('Cooldown state table does not exist')
+        output << '✗ Table DOES NOT EXIST'
+        output << '  Run: rails generate solid_queue_autoscaler:migration && rails db:migrate'
+        output << '  ⚠ Cooldowns are NOT shared across workers (using in-memory fallback)'
+      end
+
+      # 2. Events table
+      output << ''
+      output << '-' * 60
+      output << '2. EVENTS TABLE (solid_queue_autoscaler_events)'
+      output << '-' * 60
+
+      if connection.table_exists?(:solid_queue_autoscaler_events)
+        result.events_table_exists = true
+        output << '✓ Table exists'
+
+        columns = connection.columns(:solid_queue_autoscaler_events).map(&:name)
+        expected = %w[id worker_name action from_workers to_workers reason queue_depth latency_seconds metrics_json dry_run created_at]
+        missing = expected - columns
+
+        if missing.empty?
+          result.events_table_columns_ok = true
+          output << '  ✓ All expected columns present'
+        else
+          result.events_table_columns_ok = false
+          result.add_warning("Events table missing columns: #{missing.join(', ')}")
+          output << "  ⚠ Missing columns: #{missing.join(', ')}"
+        end
+
+        event_count = connection.select_value('SELECT COUNT(*) FROM solid_queue_autoscaler_events').to_i
+        output << "  Total events: #{event_count}"
+      else
+        result.events_table_exists = false
+        result.add_error('Events table does not exist')
+        output << '✗ Table DOES NOT EXIST'
+        output << '  Run: rails generate solid_queue_autoscaler:migration && rails db:migrate'
+        output << '  ⚠ Scale events are NOT being recorded (dashboard will be empty)'
+      end
+
+      # 3. Configuration
+      output << ''
+      output << '-' * 60
+      output << '3. CONFIGURATION'
+      output << '-' * 60
+
+      begin
+        result.config_valid = true
+        output << '✓ Configuration loaded'
+        output << "  enabled: #{cfg.enabled?}"
+        output << "  dry_run: #{cfg.dry_run?}"
+        output << "  persist_cooldowns: #{cfg.respond_to?(:persist_cooldowns) ? cfg.persist_cooldowns : '(not supported in this version)'}"
+        output << "  record_events: #{cfg.respond_to?(:record_events) ? cfg.record_events : '(not supported in this version)'}"
+        output << "  min_workers: #{cfg.min_workers}"
+        output << "  max_workers: #{cfg.max_workers}"
+        output << "  job_queue: #{cfg.job_queue}"
+        output << "  adapter: #{cfg.adapter.class.name}"
+      rescue StandardError => e
+        result.config_valid = false
+        result.add_error("Configuration error: #{e.message}")
+        output << "✗ Configuration error: #{e.message}"
+      end
+
+      # 4. Adapter connectivity
+      output << ''
+      output << '-' * 60
+      output << '4. ADAPTER CONNECTIVITY'
+      output << '-' * 60
+
+      begin
+        workers = cfg.adapter.current_workers
+        result.adapter_connected = true
+        output << "✓ Adapter connected (current workers: #{workers})"
+      rescue StandardError => e
+        result.adapter_connected = false
+        result.add_error("Adapter connection failed: #{e.message}")
+        output << "✗ Adapter connection failed: #{e.message}"
+      end
+
+      # 5. Solid Queue tables
+      output << ''
+      output << '-' * 60
+      output << '5. SOLID QUEUE TABLES'
+      output << '-' * 60
+
+      sq_tables = %w[solid_queue_jobs solid_queue_ready_executions solid_queue_claimed_executions solid_queue_processes]
+      result.solid_queue_tables = {}
+
+      sq_tables.each do |table|
+        if connection.table_exists?(table)
+          count = connection.select_value("SELECT COUNT(*) FROM #{table}").to_i
+          result.solid_queue_tables[table] = count
+          output << "✓ #{table}: #{count} records"
+        else
+          result.solid_queue_tables[table] = nil
+          output << "✗ #{table}: MISSING"
+        end
+      end
+
+      # Summary
+      output << ''
+      output << '=' * 60
+      output << 'SUMMARY'
+      output << '=' * 60
+
+      if result.ok?
+        output << '✓ All checks passed! Autoscaler is correctly configured.'
+        if result.cooldowns_shared?
+          output << '  Cooldowns: SHARED across workers (database-persisted)'
+        else
+          output << '  Cooldowns: In-memory only (not shared across workers)'
+        end
+        if result.events_table_exists
+          output << '  Events: RECORDING to database'
+        else
+          output << '  Events: NOT recording (events table missing)'
+        end
+      else
+        output << '⚠ Some issues found:'
+        result.errors.each { |err| output << "  ✗ #{err}" }
+        result.warnings.each { |warn| output << "  ⚠ #{warn}" }
+        output << ''
+        output << 'To fix missing tables, run:'
+        output << '  rails generate solid_queue_autoscaler:migration'
+        output << '  rails db:migrate'
+      end
+
+      puts output.join("\n") if verbose
+
+      nil
+    end
+
+    # Convenience alias so users can call verify_install! as requested
+    def verify_install!(name = :default, verbose: true)
+      verify_setup!(name, verbose: verbose)
+    end
+  end
+
+  # Structured result from verify_setup!/verify_install!
+  class VerificationResult
+    attr_accessor :connection_type,
+                  :state_table_exists, :state_table_columns_ok,
+                  :events_table_exists, :events_table_columns_ok,
+                  :config_valid, :adapter_connected,
+                  :solid_queue_tables
+
+    def initialize
+      @errors = []
+      @warnings = []
+      @solid_queue_tables = {}
+    end
+
+    def errors
+      @errors
+    end
+
+    def warnings
+      @warnings
+    end
+
+    def add_error(message)
+      @errors << message
+    end
+
+    def add_warning(message)
+      @warnings << message
+    end
+
+    def ok?
+      @errors.empty?
+    end
+
+    def tables_exist?
+      state_table_exists && events_table_exists
+    end
+
+    def cooldowns_shared?
+      state_table_exists && state_table_columns_ok
+    end
+
+    def to_h
+      {
+        ok: ok?,
+        connection_type: connection_type,
+        state_table: { exists: state_table_exists, columns_ok: state_table_columns_ok },
+        events_table: { exists: events_table_exists, columns_ok: events_table_columns_ok },
+        config_valid: config_valid,
+        adapter_connected: adapter_connected,
+        solid_queue_tables: solid_queue_tables,
+        errors: errors,
+        warnings: warnings
+      }
+    end
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solid_queue_autoscaler
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.9
 platform: ruby
 authors:
 - reillyse
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-01-16 00:00:00.000000000 Z
+date: 2026-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement