solid_queue_autoscaler 1.0.0

data/lib/solid_queue_autoscaler/dashboard/views/solid_queue_heroku_autoscaler/dashboard/workers/show.html.erb

@@ -0,0 +1,209 @@
+<div class="d-flex justify-between align-center mb-3">
+  <h2>
+    Worker: <%= @worker[:name] %>
+    <% if @worker[:enabled] %>
+      <span class="badge badge-success">Active</span>
+    <% else %>
+      <span class="badge badge-neutral">Disabled</span>
+    <% end %>
+    <% if @worker[:dry_run] %>
+      <span class="badge badge-warning">Dry Run</span>
+    <% end %>
+  </h2>
+  <div class="d-flex gap-2">
+    <%= link_to '← Back', workers_path, class: 'btn btn-secondary btn-sm' %>
+    <%= button_to 'Scale Now', scale_worker_path(@worker[:name]), method: :post, class: 'btn btn-primary btn-sm', disabled: !@worker[:enabled] %>
+  </div>
+</div>
+
+<div class="grid grid-2 mb-3">
+  <!-- Current Status -->
+  <div class="card">
+    <h3>Current Status</h3>
+    
+    <div class="d-flex justify-between align-center mb-2">
+      <div>
+        <strong class="text-info" style="font-size: 2rem;"><%= @worker[:current_workers] %></strong>
+        <span class="text-muted">/ <%= @worker[:max_workers] %> workers</span>
+      </div>
+      <div class="text-muted">
+        Min: <%= @worker[:min_workers] %> | Max: <%= @worker[:max_workers] %>
+      </div>
+    </div>
+    
+    <div class="progress-bar">
+      <% pct = (@worker[:current_workers].to_f / @worker[:max_workers] * 100).round %>
+      <div class="fill" style="width: <%= pct %>%; background: <%= pct > 80 ? 'var(--warning)' : 'var(--success)' %>;"></div>
+    </div>
+    
+    <div class="grid grid-2 mt-2">
+      <div>
+        <div class="text-muted">Process Type</div>
+        <code><%= @worker[:process_type] %></code>
+      </div>
+      <div>
+        <div class="text-muted">Strategy</div>
+        <code><%= @worker[:scaling_strategy] %></code>
+      </div>
+      <div>
+        <div class="text-muted">Queues</div>
+        <code><%= Array(@worker[:queues]).join(', ') %></code>
+      </div>
+    </div>
+  </div>
+
+  <!-- Metrics -->
+  <div class="card">
+    <h3>Current Metrics</h3>
+    
+    <div class="grid grid-2">
+      <div class="mb-2">
+        <div class="text-muted">Queue Depth</div>
+        <strong class="<%= @worker[:metrics][:queue_depth] > @worker[:thresholds][:scale_up_queue_depth] ? 'text-danger' : '' %>" style="font-size: 1.5rem;">
+          <%= @worker[:metrics][:queue_depth] %>
+        </strong>
+        <div class="text-muted" style="font-size: 0.8rem;">
+          ↑ threshold: <%= @worker[:thresholds][:scale_up_queue_depth] %> | ↓ threshold: <%= @worker[:thresholds][:scale_down_queue_depth] %>
+        </div>
+      </div>
+      
+      <div class="mb-2">
+        <div class="text-muted">Oldest Job Age</div>
+        <strong class="<%= @worker[:metrics][:latency_seconds] > @worker[:thresholds][:scale_up_latency] ? 'text-danger' : '' %>" style="font-size: 1.5rem;">
+          <%= @worker[:metrics][:latency_seconds].round %>s
+        </strong>
+        <div class="text-muted" style="font-size: 0.8rem;">
+          ↑ threshold: <%= @worker[:thresholds][:scale_up_latency] %>s | ↓ threshold: <%= @worker[:thresholds][:scale_down_latency] %>s
+        </div>
+      </div>
+      
+      <div>
+        <div class="text-muted">Jobs/Minute</div>
+        <strong style="font-size: 1.5rem;"><%= @worker[:metrics][:jobs_per_minute] %></strong>
+      </div>
+      
+      <div>
+        <div class="text-muted">Claimed Jobs</div>
+        <strong style="font-size: 1.5rem;"><%= @worker[:metrics][:claimed_jobs] %></strong>
+      </div>
+      
+      <div>
+        <div class="text-muted">Failed Jobs</div>
+        <strong class="<%= @worker[:metrics][:failed_jobs] > 0 ? 'text-danger' : '' %>" style="font-size: 1.5rem;">
+          <%= @worker[:metrics][:failed_jobs] %>
+        </strong>
+      </div>
+      
+      <div>
+        <div class="text-muted">Active Workers</div>
+        <strong style="font-size: 1.5rem;"><%= @worker[:metrics][:active_workers] %></strong>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Cooldowns -->
+<div class="card mb-3">
+  <h3>Cooldown Status</h3>
+  
+  <div class="grid grid-2">
+    <div>
+      <div class="text-muted mb-1">Scale Up Cooldown</div>
+      <% if @worker[:cooldowns][:scale_up_remaining] > 0 %>
+        <span class="badge badge-info">
+          <%= @worker[:cooldowns][:scale_up_remaining] %>s remaining
+        </span>
+      <% else %>
+        <span class="text-success">Ready</span>
+      <% end %>
+      <% if @worker[:cooldowns][:last_scale_up] %>
+        <div class="text-muted" style="font-size: 0.8rem;">
+          Last: <%= @worker[:cooldowns][:last_scale_up].strftime('%Y-%m-%d %H:%M:%S') %>
+        </div>
+      <% end %>
+    </div>
+    
+    <div>
+      <div class="text-muted mb-1">Scale Down Cooldown</div>
+      <% if @worker[:cooldowns][:scale_down_remaining] > 0 %>
+        <span class="badge badge-warning">
+          <%= @worker[:cooldowns][:scale_down_remaining] %>s remaining
+        </span>
+      <% else %>
+        <span class="text-success">Ready</span>
+      <% end %>
+      <% if @worker[:cooldowns][:last_scale_down] %>
+        <div class="text-muted" style="font-size: 0.8rem;">
+          Last: <%= @worker[:cooldowns][:last_scale_down].strftime('%Y-%m-%d %H:%M:%S') %>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</div>
+
+<!-- Recent Events -->
+<div class="card">
+  <div class="d-flex justify-between align-center mb-2">
+    <h3>Recent Events</h3>
+    <%= link_to 'View All', events_path(worker: @worker[:name]), class: 'btn btn-secondary btn-sm' %>
+  </div>
+
+  <% if !events_available? %>
+    <p class="text-muted">
+      Events table not found. Run:
+      <code>rails generate solid_queue_autoscaler:dashboard</code>
+    </p>
+  <% elsif @events.empty? %>
+    <p class="text-muted">No scale events recorded for this worker.</p>
+  <% else %>
+    <table>
+      <thead>
+        <tr>
+          <th>Time</th>
+          <th>Action</th>
+          <th>Workers</th>
+          <th>Queue Depth</th>
+          <th>Latency</th>
+          <th>Reason</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @events.each do |event| %>
+          <tr>
+            <td>
+              <div><%= event.created_at.strftime('%Y-%m-%d') %></div>
+              <div class="time-ago"><%= event.created_at.strftime('%H:%M:%S') %></div>
+            </td>
+            <td>
+              <% case event.action %>
+              <% when 'scale_up' %>
+                <span class="badge badge-success">↑ Scale Up</span>
+              <% when 'scale_down' %>
+                <span class="badge badge-warning">↓ Scale Down</span>
+              <% when 'no_change' %>
+                <span class="badge badge-neutral">— No Change</span>
+              <% when 'skipped' %>
+                <span class="badge badge-info">⏭ Skipped</span>
+              <% when 'error' %>
+                <span class="badge badge-danger">✕ Error</span>
+              <% end %>
+              <% if event.dry_run %>
+                <span class="badge badge-neutral">DRY</span>
+              <% end %>
+            </td>
+            <td>
+              <% if event.scaled? %>
+                <strong><%= event.from_workers %></strong> → <strong><%= event.to_workers %></strong>
+              <% else %>
+                <%= event.from_workers %>
+              <% end %>
+            </td>
+            <td><%= event.queue_depth %></td>
+            <td><%= event.latency_seconds.round %>s</td>
+            <td class="text-muted"><%= event.reason %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  <% end %>
+</div>

data/lib/solid_queue_autoscaler/dashboard.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require_relative 'dashboard/engine' if defined?(Rails::Engine)
+
+module SolidQueueAutoscaler
+  # Dashboard module provides a web UI for monitoring the autoscaler.
+  # Integrates with Mission Control Solid Queue when available.
+  module Dashboard
+    class << self
+      # Returns current autoscaler status for all workers
+      # @return [Hash] Status information for all workers
+      def status
+        workers = SolidQueueAutoscaler.registered_workers
+        workers = [:default] if workers.empty?
+
+        workers.each_with_object({}) do |name, status|
+          status[name] = worker_status(name)
+        end
+      end
+
+      # Returns status for a specific worker
+      # @param name [Symbol] Worker name
+      # @return [Hash] Status information
+      def worker_status(name)
+        config = SolidQueueAutoscaler.config(name)
+        metrics = safe_metrics(name)
+        tracker = CooldownTracker.new(config: config, key: name.to_s)
+
+        {
+          name: name,
+          enabled: config.enabled?,
+          dry_run: config.dry_run?,
+          current_workers: safe_current_workers(name),
+          min_workers: config.min_workers,
+          max_workers: config.max_workers,
+          queues: config.queues || ['all'],
+          process_type: config.process_type,
+          scaling_strategy: config.scaling_strategy,
+          metrics: {
+            queue_depth: metrics&.queue_depth || 0,
+            latency_seconds: metrics&.oldest_job_age_seconds || 0,
+            jobs_per_minute: metrics&.jobs_per_minute || 0,
+            claimed_jobs: metrics&.claimed_jobs || 0,
+            failed_jobs: metrics&.failed_jobs || 0,
+            active_workers: metrics&.active_workers || 0
+          },
+          cooldowns: {
+            scale_up_remaining: tracker.scale_up_cooldown_remaining.round,
+            scale_down_remaining: tracker.scale_down_cooldown_remaining.round,
+            last_scale_up: tracker.last_scale_up_at,
+            last_scale_down: tracker.last_scale_down_at
+          },
+          thresholds: {
+            scale_up_queue_depth: config.scale_up_queue_depth,
+            scale_up_latency: config.scale_up_latency_seconds,
+            scale_down_queue_depth: config.scale_down_queue_depth,
+            scale_down_latency: config.scale_down_latency_seconds
+          }
+        }
+      end
+
+      # Returns recent scale events
+      # @param limit [Integer] Maximum events to return
+      # @param worker_name [String, nil] Filter by worker
+      # @return [Array<ScaleEvent>] Recent events
+      def recent_events(limit: 50, worker_name: nil)
+        ScaleEvent.recent(limit: limit, worker_name: worker_name)
+      end
+
+      # Returns event statistics
+      # @param since [Time] Start time
+      # @param worker_name [String, nil] Filter by worker
+      # @return [Hash] Statistics
+      def event_stats(since: 24.hours.ago, worker_name: nil)
+        ScaleEvent.stats(since: since, worker_name: worker_name)
+      end
+
+      # Checks if the events table is available
+      # @return [Boolean] True if events can be recorded
+      def events_table_available?
+        ScaleEvent.table_exists?
+      end
+
+      private
+
+      def safe_metrics(name)
+        SolidQueueAutoscaler.metrics(name)
+      rescue StandardError
+        nil
+      end
+
+      def safe_current_workers(name)
+        SolidQueueAutoscaler.current_workers(name)
+      rescue StandardError
+        0
+      end
+    end
+  end
+end

data/lib/solid_queue_autoscaler/decision_engine.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+module SolidQueueAutoscaler
+  class DecisionEngine
+    Decision = Struct.new(:action, :from, :to, :reason, keyword_init: true) do
+      def scale_up?
+        action == :scale_up
+      end
+
+      def scale_down?
+        action == :scale_down
+      end
+
+      def no_change?
+        action == :no_change
+      end
+
+      def delta
+        to - from
+      end
+    end
+
+    def initialize(config: nil)
+      @config = config || SolidQueueAutoscaler.config
+    end
+
+    def decide(metrics:, current_workers:)
+      return no_change_decision(current_workers, 'Autoscaler is disabled') unless @config.enabled?
+
+      if should_scale_up?(metrics, current_workers)
+        scale_up_decision(metrics, current_workers)
+      elsif should_scale_down?(metrics, current_workers)
+        scale_down_decision(metrics, current_workers)
+      else
+        no_change_decision(current_workers, determine_no_change_reason(metrics, current_workers))
+      end
+    end
+
+    private
+
+    def should_scale_up?(metrics, current_workers)
+      return false if current_workers >= @config.max_workers
+
+      queue_depth_high = metrics.queue_depth >= @config.scale_up_queue_depth
+      latency_high = metrics.oldest_job_age_seconds >= @config.scale_up_latency_seconds
+
+      queue_depth_high || latency_high
+    end
+
+    def should_scale_down?(metrics, current_workers)
+      return false if current_workers <= @config.min_workers
+
+      queue_depth_low = metrics.queue_depth <= @config.scale_down_queue_depth
+      latency_low = metrics.oldest_job_age_seconds <= @config.scale_down_latency_seconds
+      is_idle = metrics.idle?
+
+      (queue_depth_low && latency_low) || is_idle
+    end
+
+    def scale_up_decision(metrics, current_workers)
+      target = calculate_scale_up_target(metrics, current_workers)
+      reason = build_scale_up_reason(metrics, current_workers, target)
+
+      Decision.new(
+        action: :scale_up,
+        from: current_workers,
+        to: target,
+        reason: reason
+      )
+    end
+
+    def scale_down_decision(metrics, current_workers)
+      target = calculate_scale_down_target(metrics, current_workers)
+      reason = build_scale_down_reason(metrics, current_workers, target)
+
+      Decision.new(
+        action: :scale_down,
+        from: current_workers,
+        to: target,
+        reason: reason
+      )
+    end
+
+    def calculate_scale_up_target(metrics, current_workers)
+      raw_target = case @config.scaling_strategy
+                   when :proportional
+                     calculate_proportional_scale_up_target(metrics, current_workers)
+                   when :step_function
+                     calculate_step_function_target(metrics, current_workers)
+                   else # :fixed
+                     current_workers + @config.scale_up_increment
+                   end
+
+      [raw_target, @config.max_workers].min
+    end
+
+    def calculate_scale_down_target(metrics, current_workers)
+      raw_target = case @config.scaling_strategy
+                   when :proportional
+                     calculate_proportional_scale_down_target(metrics, current_workers)
+                   when :step_function
+                     calculate_step_function_target(metrics, current_workers)
+                   else # :fixed
+                     current_workers - @config.scale_down_decrement
+                   end
+
+      [raw_target, @config.min_workers].max
+    end
+
+    def calculate_proportional_scale_up_target(metrics, current_workers)
+      # Calculate workers needed based on queue depth
+      jobs_over_threshold = [metrics.queue_depth - @config.scale_up_queue_depth, 0].max
+      workers_for_depth = (jobs_over_threshold.to_f / @config.scale_up_jobs_per_worker).ceil
+
+      # Calculate workers needed based on latency
+      latency_over_threshold = [metrics.oldest_job_age_seconds - @config.scale_up_latency_seconds, 0].max
+      workers_for_latency = (latency_over_threshold / @config.scale_up_latency_per_worker).ceil
+
+      # Take the higher of the two calculations
+      additional_workers = [workers_for_depth, workers_for_latency].max
+
+      # Always add at least scale_up_increment if we're scaling up
+      additional_workers = [@config.scale_up_increment, additional_workers].max
+
+      current_workers + additional_workers
+    end
+
+    def calculate_proportional_scale_down_target(metrics, current_workers)
+      # If idle, scale down aggressively
+      return @config.min_workers if metrics.idle?
+
+      # Calculate how much capacity we have based on queue depth
+      jobs_under_capacity = [@config.scale_down_queue_depth - metrics.queue_depth, 0].max
+      workers_to_remove = (jobs_under_capacity.to_f / @config.scale_down_jobs_per_worker).floor
+
+      # Ensure we remove at least scale_down_decrement if we're scaling down
+      workers_to_remove = [@config.scale_down_decrement, workers_to_remove].max
+
+      current_workers - workers_to_remove
+    end
+
+    def calculate_step_function_target(metrics, current_workers)
+      # Step function uses fixed thresholds (future implementation)
+      # For now, fall back to fixed strategy
+      if should_scale_up?(metrics, current_workers)
+        current_workers + @config.scale_up_increment
+      else
+        current_workers - @config.scale_down_decrement
+      end
+    end
+
+    def no_change_decision(current_workers, reason)
+      Decision.new(
+        action: :no_change,
+        from: current_workers,
+        to: current_workers,
+        reason: reason
+      )
+    end
+
+    def build_scale_up_reason(metrics, current_workers = nil, target = nil)
+      reasons = []
+
+      if metrics.queue_depth >= @config.scale_up_queue_depth
+        reasons << "queue_depth=#{metrics.queue_depth} >= #{@config.scale_up_queue_depth}"
+      end
+
+      if metrics.oldest_job_age_seconds >= @config.scale_up_latency_seconds
+        reasons << "latency=#{metrics.oldest_job_age_seconds.round}s >= #{@config.scale_up_latency_seconds}s"
+      end
+
+      base_reason = reasons.join(', ')
+
+      if @config.scaling_strategy == :proportional && current_workers && target
+        delta = target - current_workers
+        "#{base_reason} [proportional: +#{delta} workers]"
+      else
+        base_reason
+      end
+    end
+
+    def build_scale_down_reason(metrics, current_workers = nil, target = nil)
+      if metrics.idle?
+        base_reason = 'queue is idle (no pending or claimed jobs)'
+      else
+        reasons = []
+
+        if metrics.queue_depth <= @config.scale_down_queue_depth
+          reasons << "queue_depth=#{metrics.queue_depth} <= #{@config.scale_down_queue_depth}"
+        end
+
+        if metrics.oldest_job_age_seconds <= @config.scale_down_latency_seconds
+          reasons << "latency=#{metrics.oldest_job_age_seconds.round}s <= #{@config.scale_down_latency_seconds}s"
+        end
+
+        base_reason = reasons.join(', ')
+      end
+
+      if @config.scaling_strategy == :proportional && current_workers && target
+        delta = current_workers - target
+        "#{base_reason} [proportional: -#{delta} workers]"
+      else
+        base_reason
+      end
+    end
+
+    def determine_no_change_reason(metrics, current_workers)
+      # Check if we would scale up but we're at max
+      queue_depth_high = metrics.queue_depth >= @config.scale_up_queue_depth
+      latency_high = metrics.oldest_job_age_seconds >= @config.scale_up_latency_seconds
+      would_scale_up = queue_depth_high || latency_high
+
+      # Check if we would scale down but we're at min
+      queue_depth_low = metrics.queue_depth <= @config.scale_down_queue_depth
+      latency_low = metrics.oldest_job_age_seconds <= @config.scale_down_latency_seconds
+      is_idle = metrics.idle?
+      would_scale_down = (queue_depth_low && latency_low) || is_idle
+
+      if current_workers >= @config.max_workers && would_scale_up
+        "at max_workers (#{@config.max_workers})"
+      elsif current_workers <= @config.min_workers && would_scale_down
+        "at min_workers (#{@config.min_workers})"
+      else
+        "metrics within normal range (depth=#{metrics.queue_depth}, latency=#{metrics.oldest_job_age_seconds.round}s)"
+      end
+    end
+  end
+end

data/lib/solid_queue_autoscaler/errors.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SolidQueueAutoscaler
+  # Base error class for all autoscaler errors.
+  class Error < StandardError; end
+
+  # Raised when configuration is invalid.
+  class ConfigurationError < Error; end
+
+  # Raised when the advisory lock cannot be acquired.
+  class LockError < Error; end
+
+  # Raised when Heroku API calls fail.
+  class HerokuAPIError < Error
+    attr_reader :status_code, :response_body
+
+    def initialize(message, status_code: nil, response_body: nil)
+      super(message)
+      @status_code = status_code
+      @response_body = response_body
+    end
+  end
+
+  # Raised when Kubernetes API calls fail.
+  class KubernetesAPIError < Error
+    attr_reader :original_error
+
+    def initialize(message, original_error: nil)
+      super(message)
+      @original_error = original_error
+    end
+  end
+
+  class MetricsError < Error; end
+
+  class CooldownActiveError < Error
+    attr_reader :remaining_seconds
+
+    def initialize(remaining_seconds)
+      @remaining_seconds = remaining_seconds
+      super("Cooldown active, #{remaining_seconds.round}s remaining")
+    end
+  end
+end