rails_autoscale_agent 0.9.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd7ffb8cd27de3a23d5e76bd9a9c7768ce1fd397389291285022523fb398ee45
-  data.tar.gz: d8cf3fa7ab2994ef196742fe872170c069dc6f14bc0490b595ff175e48ce59e6
+  metadata.gz: f313cab3eefd93584c9d2484698d836ad19467b2e03cd26623003edcd1c64ad5
+  data.tar.gz: d8b33799b6dc0c2e92b6fbe770de41ac7e104a8d887965cf35538ea9ab6b8acb
 SHA512:
-  metadata.gz: '0976e83784dd6a622b18a67491e727cbe9f0fb1a54e1974f50b61f441f642a296b5a4e3349e2f4d292acc35c844765527c16e097cf7ac914a7224b68e2ae8bd6'
-  data.tar.gz: 2de7d8a61d44fd408cfbab88b1be37327bd5836da8c125f4c61f80ac7503c0a44c5ee5f893919eb5487e4a34a817102a68aabd76699d34d6ce3086680b8aceb7
+  metadata.gz: 3208e51e66e87a9c5e076322c04f7f3594295d11ad2e5c16cdea5600372e02b70e50e15138788fe5d59a7d6854a8c59c908db5cdb28878386989c7e8fb2d9049
+  data.tar.gz: 1b771752c5fde1c3cd534a4ef2aca0e5917d3343d10e97cfb5fdb64a89f72cd5343e4662af281c214a35459cb77759f7c19b5110fd8bed979b54ffeff97a1de0

data/CHANGELOG.md

@@ -0,0 +1,121 @@
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.11.0...master)
+
+No currently unreleased changes.
+
+## [0.11.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.10.2...v0.11.0)
+
+### Added
+
+- Add `RAILS_AUTOSCALE_MAX_QUEUES` config option. ([28738a5](https://github.com/adamlogic/rails_autoscale_agent/commit/28738a5dc4cd6b0a46e77459d6f98e6b33072da9))
+
+## [0.10.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.10.1...v0.10.2) - 2021-01-12
+
+### Changed
+
+- Loosen Ruby constraint to allow Ruby 3. ([#36](https://github.com/adamlogic/rails_autoscale_agent/pull/36))
+
+## [0.10.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.1...v0.10.1) - 2021-01-03
+
+### Added
+
+- Add support for [long-running jobs](https://railsautoscale.com/docs/long-running-jobs/) in Sidekiq and Delayed Job.
+- Handle x-request-start measured in seconds (instead of milliseconds) to support nginx buildpack ([cd092f3](https://github.com/adamlogic/rails_autoscale_agent/commit/cd092f38718abf5ffaea866bcae7831d4c910ffd))
+- Override worker adapter config via env var ([75dd06b](https://github.com/adamlogic/rails_autoscale_agent/commit/75dd06b2a7ff4eeab829eec24d503dc067c8fe32))
+
+### Changed
+
+- Require Ruby 2.5 or newer. ([b033050](https://github.com/adamlogic/rails_autoscale_agent/commit/b033050b7f9d4d7f1e50dbd780cf0e1822249268))
+- Only report worker metrics from web.1 to avoid redundant data. ([d5d5fa8](https://github.com/adamlogic/rails_autoscale_agent/commit/d5d5fa87fb4d7d046832a64edde9ed0c3a6ec75f))
+- Don't collect worker metrics for an unreasonable number of queues. ([a9358af](https://github.com/adamlogic/rails_autoscale_agent/commit/a9358af74a29a941d1f1d60a0222077dafd5ce08))
+
+### Fixed
+
+- Avoid holding onto database connections (DJ & Que only). ([3919ca5](https://github.com/adamlogic/rails_autoscale_agent/commit/3919ca54420cafa82abf9f8cd251569f9637482b))
+- Better error handling for worker adapters. ([190786e](https://github.com/adamlogic/rails_autoscale_agent/commit/190786e4a910d41e394a3129aac1d23b594dbd9b))
+- Don't collect metrics of the reporter isn't running. Avoids memory bloat. ([247c322](https://github.com/adamlogic/rails_autoscale_agent/commit/247c322cffc625a8c6b2395080a048ffb94e7f3b))
+
+## [0.10.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.1...v0.10.0) - 2021-01-03 [YANKED]
+
+_I released the wrong branch 🤦‍♂️_
+
+## [0.9.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.0...v0.9.1) - 2020-07-29
+
+### Fixed
+
+- Fix a bug in error handling. ([3018542](https://github.com/adamlogic/rails_autoscale_agent/commit/3018542cd046fc4e1bd6e7da86e72a6aa2d50a8f))
+- Remove unintentional Rails dependency.
+
+## [0.9.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.3...v0.9.0) - 2020-07-12
+
+### Added
+
+- Add support for Resque workers.
+- Add dev mode for working on the agent gem itself. ([47e3fca](https://github.com/adamlogic/rails_autoscale_agent/commit/47e3fca5b788f48567a345d9cab3a26b9cd87693))
+- Report agent exceptions to Rails Autoscale.
+
+### Changed
+
+- Adjust queue time metric to exclude time waiting for large request bodies. ([#25](https://github.com/adamlogic/rails_autoscale_agent/pull/25))
+- Que and DJ jobs without a queue name will be included in the "default" queue metrics.
+
+### Fixed
+
+- Multiple fixes to the Delayed Job SQL query.
+
+## [0.8.3](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.2...v0.8.3) - 2020-05-26
+
+### Fixed
+
+- Ignored failed job in Delayed Job adapter. ([fa72fc2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.2...v0.8.3))
+
+## [0.8.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.1...v0.8.2) - 2020-05-22
+
+### Fixed
+
+- Ignore worker metrics from unnamed queues (DJ & Que only). These metrics were being lumped with web metrics. ([#21](https://github.com/adamlogic/rails_autoscale_agent/pull/21))
+
+## [0.8.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.0...v0.8.1) - 2020-05-04
+
+### Fixed
+
+- Ignore failed jobs in Que adapter. ([#18](https://github.com/adamlogic/rails_autoscale_agent/pull/18))
+
+## [0.8.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.7.0...v0.8.0) - 2020-03-21
+
+### Added
+
+- Add support for Delayed Job ([#14](https://github.com/adamlogic/rails_autoscale_agent/pull/14))
+- Add support for Que ([#15](https://github.com/adamlogic/rails_autoscale_agent/pull/15))
+
+## [0.7.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.3...v0.7.0) - 2019-12-04
+
+### Added
+
+- Make worker adapters configurable. ([012d937](https://github.com/adamlogic/rails_autoscale_agent/commit/012d9379296763f5e42df95f05b066fe82ab0051))
+
+## [0.6.3](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.2...v0.6.3) - 2019-06-25
+
+### Fixed
+
+- Fix issues with logging.
+
+## [0.6.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.1...v0.6.2) - 2019-06-22
+
+### Fixed
+
+- Fix issues with logging.
+
+## [0.6.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.0...v0.6.1) - 2019-05-06
+
+### Fixed
+
+- Don't assume Sidekiq is present.
+
+## [0.6.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.4.1...v0.6.0) - 2019-05-03
+
+### Added
+
+- Add support for autoscaling Sidekiq.

data/README.md

@@ -6,7 +6,8 @@ This gem works together with the [Rails Autoscale](https://railsautoscale.com) H
 
 ## Requirements
 
-Tested with Rails versions 3.2 and higher and Ruby versions 1.9.3 and higher.
+- Rack-based app
+- Ruby 2.5 or newer
 
 ## Getting Started
 
@@ -22,7 +23,7 @@ The agent will only communicate with Rails Autoscale if a `RAILS_AUTOSCALE_URL`
 
 ## Non-Rails Rack apps
 
-You'll need to insert the `RailsAutoscaleAgent::Middleware` manually. Insert it before `Rack::Runtime` to ensure accuracy of request queue timings.
+You'll need to `require 'rails_autoscale_agent/middleware'` and insert the `RailsAutoscaleAgent::Middleware` manually. Insert it before `Rack::Runtime` to ensure accuracy of request queue timings.
 
 ## What data is collected?
 
@@ -37,6 +38,33 @@ The middleware agent runs in its own thread so your web requests are not impacte
 
 Rails Autoscale aggregates and stores this information to power the autoscaling algorithm and dashboard visualizations.
 
+## Configuration
+
+Most Rails Autoscale configurations are handled via the settings page on your Rails Autoscale dashboard, but there a few ways you can directly change the behavior of the agent via environment variables:
+
+- `RAILS_AUTOSCALE_DEBUG` - Enables debug logging. See more in the [logging](#logging) section below.
+- `RAILS_AUTOSCALE_WORKER_ADAPTER` - Overrides the available worker adapters. See more in the [worker adapters](#worker-adapters) section below.
+- `RAILS_AUTOSCALE_LONG_JOBS` - Enables reporting for active workers. See [Handling Long-Running Background Jobs](https://railsautoscale.com/docs/long-running-jobs/) in the Rails Autoscale docs for more.
+- `RAILS_AUTOSCALE_MAX_QUEUES` - Worker metrics will only report up to 50 queues by default. If you have more than 50 queues, you'll need to configure this settings or reduce your number of queues.
+
+## Worker adapters
+
+Rails Autoscale supports autoscaling worker dynos. Out of the box, four job backends are supported: Sidekiq, Resque, Delayed Job, and Que. The agent will automatically enable the appropriate worker adapter based on what you have installed in your app.
+
+In some scenarios you might want to override this behavior. Let's say you have both Sidekiq and Resque installed 🤷‍♂️, but you only want Rails Autoscale to collect metrics for Sidekiq. Here's how you'd override that:
+
+```
+heroku config:add RAILS_AUTOSCALE_WORKER_ADAPTER=sidekiq
+```
+
+You can also disable collection of worker metrics altogether:
+
+```
+heroku config:add RAILS_AUTOSCALE_WORKER_ADAPTER=""
+```
+
+It's also possible to write a custom worker adapter. See [these docs](https://railsautoscale.com/docs/custom-worker-adapter/) for details.
+
 ## Troubleshooting
 
 Once installed, you should see something like this in your development log:

data/lib/rails_autoscale_agent/autoscale_api.rb

@@ -59,7 +59,7 @@ module RailsAutoscaleAgent
 
       case response.code.to_i
       when 200...300 then SuccessResponse.new(response.body)
-      else FailureResponse.new(response.message)
+      else FailureResponse.new([response.code, response.message].join(' - '))
       end
     end
 

data/lib/rails_autoscale_agent/config.rb

@@ -4,29 +4,27 @@ require 'singleton'
 
 module RailsAutoscaleAgent
   class Config
+    DEFAULT_WORKER_ADAPTERS = 'sidekiq,delayed_job,que,resque'
+
     include Singleton
 
     attr_accessor :report_interval, :logger, :api_base_url, :max_request_size,
-                  :dyno, :pid, :addon_name, :worker_adapters, :dev_mode, :debug, :quiet
+                  :dyno, :addon_name, :worker_adapters, :dev_mode, :debug, :quiet,
+                  :track_long_running_jobs, :max_queues,
+
+                  # legacy configs, no longer used
+                  :sidekiq_latency_for_active_jobs, :latency_for_active_jobs
 
     def initialize
-      require 'rails_autoscale_agent/worker_adapters/sidekiq'
-      require 'rails_autoscale_agent/worker_adapters/delayed_job'
-      require 'rails_autoscale_agent/worker_adapters/que'
-      require 'rails_autoscale_agent/worker_adapters/resque'
-      @worker_adapters = [
-        WorkerAdapters::Sidekiq.instance,
-        WorkerAdapters::DelayedJob.instance,
-        WorkerAdapters::Que.instance,
-        WorkerAdapters::Resque.instance,
-      ]
+      @worker_adapters = prepare_worker_adapters
 
       # Allow the add-on name to be configured - needed for testing
       @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
       @api_base_url = ENV["#{@addon_name}_URL"]
       @dev_mode = ENV['RAILS_AUTOSCALE_DEV'] == 'true'
       @debug = dev_mode? || ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
-      @pid = Process.pid
+      @track_long_running_jobs = ENV['RAILS_AUTOSCALE_LONG_JOBS'] == 'true'
+      @max_queues = ENV.fetch('RAILS_AUTOSCALE_MAX_QUEUES', 50).to_i
       @max_request_size = 100_000 # ignore request payloads over 100k since they skew the queue times
       @report_interval = 10 # this default will be overwritten during Reporter#register!
       @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
@@ -34,7 +32,7 @@ module RailsAutoscaleAgent
     end
 
     def to_s
-      "#{@dyno}##{@pid}"
+      "#{@dyno}##{Process.pid}"
     end
 
     def ignore_large_requests?
@@ -44,5 +42,16 @@ module RailsAutoscaleAgent
     alias_method :dev_mode?, :dev_mode
     alias_method :debug?, :debug
     alias_method :quiet?, :quiet
+
+    private
+
+    def prepare_worker_adapters
+      adapter_names = (ENV['RAILS_AUTOSCALE_WORKER_ADAPTER'] || DEFAULT_WORKER_ADAPTERS).split(',')
+      adapter_names.map do |adapter_name|
+        require "rails_autoscale_agent/worker_adapters/#{adapter_name}"
+        adapter_constant_name = adapter_name.capitalize.gsub(/(?:_)(.)/i) { $1.upcase }
+        WorkerAdapters.const_get(adapter_constant_name).instance
+      end
+    end
   end
 end

data/lib/rails_autoscale_agent/registration.rb

@@ -8,7 +8,7 @@ module RailsAutoscaleAgent
     def to_params
       {
         dyno: config.dyno,
-        pid: config.pid,
+        pid: Process.pid,
         ruby_version: RUBY_VERSION,
         rails_version: defined?(Rails) && Rails.version,
         gem_version: VERSION,

data/lib/rails_autoscale_agent/report.rb

@@ -12,7 +12,7 @@ module RailsAutoscaleAgent
     def to_params(config)
       {
         dyno: config.dyno,
-        pid: config.pid,
+        pid: Process.pid,
       }
     end
 

data/lib/rails_autoscale_agent/reporter.rb

@@ -6,8 +6,6 @@ require 'rails_autoscale_agent/autoscale_api'
 require 'rails_autoscale_agent/time_rounder'
 require 'rails_autoscale_agent/registration'
 
-# Reporter wakes up every minute to send metrics to the RailsAutoscale API
-
 module RailsAutoscaleAgent
   class Reporter
     include Singleton
@@ -20,6 +18,7 @@ module RailsAutoscaleAgent
     def start!(config, store)
       @started = true
       @worker_adapters = config.worker_adapters.select(&:enabled?)
+      @dyno_num = config.dyno.to_s.split('.').last.to_i
 
       if !config.api_base_url && !config.dev_mode?
         logger.info "Reporter not started: #{config.addon_name}_URL is not set"
@@ -34,15 +33,14 @@ module RailsAutoscaleAgent
           multiplier = 1 - (rand / 4) # between 0.75 and 1.0
           sleep config.report_interval * multiplier
 
-          begin
-            @worker_adapters.map { |a| a.collect!(store) }
-            report!(config, store)
-          rescue => ex
-            # Exceptions in threads other than the main thread will fail silently
-            # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
-            logger.error "Reporter error: #{ex.inspect}"
-            AutoscaleApi.new(config.api_base_url).report_exception!(ex)
+          # It's redundant to report worker metrics from every web dyno, so only report from web.1
+          if @dyno_num == 1
+            @worker_adapters.map do |adapter|
+              report_exceptions(config) { adapter.collect!(store) }
+            end
           end
+
+          report_exceptions(config) { report!(config, store) }
         end
       end
     end
@@ -51,23 +49,21 @@ module RailsAutoscaleAgent
       @started
     end
 
+    private
+
     def report!(config, store)
       report = store.pop_report
 
-      if report.measurements.any?
-        logger.info "Reporting #{report.measurements.size} measurements"
+      logger.info "Reporting #{report.measurements.size} measurements"
 
-        params = report.to_params(config)
-        result = AutoscaleApi.new(config).report_metrics!(params, report.to_csv)
+      params = report.to_params(config)
+      result = AutoscaleApi.new(config).report_metrics!(params, report.to_csv)
 
-        case result
-        when AutoscaleApi::SuccessResponse
-          logger.debug "Reported successfully"
-        when AutoscaleApi::FailureResponse
-          logger.error "Reporter failed: #{result.failure_message}"
-        end
-      else
-        logger.debug "Reporter has nothing to report"
+      case result
+      when AutoscaleApi::SuccessResponse
+        logger.debug "Reported successfully"
+      when AutoscaleApi::FailureResponse
+        logger.error "Reporter failed: #{result.failure_message}"
       end
     end
 
@@ -86,5 +82,20 @@ module RailsAutoscaleAgent
         logger.error "Reporter failed to register: #{result.failure_message}"
       end
     end
+
+    def report_exceptions(config)
+      begin
+        yield
+      rescue => ex
+        # Exceptions in threads other than the main thread will fail silently
+        # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
+        logger.error "Reporter error: #{ex.inspect}"
+        AutoscaleApi.new(config).report_exception!(ex)
+      end
+    rescue => ex
+      # An exception was encountered while trying to report the original exception.
+      # Swallow the error so the reporter continues to report.
+      logger.error "Exception reporting error: #{ex.inspect}"
+    end
   end
 end

data/lib/rails_autoscale_agent/request.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'rails_autoscale_agent/logger'
+
 module RailsAutoscaleAgent
   class Request
     include Logger
@@ -9,32 +11,39 @@ module RailsAutoscaleAgent
       @id = env['HTTP_X_REQUEST_ID']
       @size = env['rack.input'].respond_to?(:size) ? env['rack.input'].size : 0
       @request_body_wait = env['puma.request_body_wait'].to_i
+      @request_start_header = env['HTTP_X_REQUEST_START']
+    end
+
+    def ignore?
+      @config.ignore_large_requests? && @size > @config.max_request_size
+    end
 
-      @entered_queue_at = if unix_millis = env['HTTP_X_REQUEST_START']
-        Time.at(unix_millis.to_f / 1000)
-      elsif config.dev_mode?
+    def started_at
+      if @request_start_header
+        # Heroku sets the header as an integer, measured in milliseconds.
+        # If nginx is involved, it might be in seconds with fractional milliseconds,
+        # and it might be preceeded by "t=". We can all cases by removing non-digits
+        # and treating as milliseconds.
+        Time.at(@request_start_header.gsub(/\D/, '').to_i / 1000.0)
+      elsif @config.dev_mode?
         # In dev mode, fake a queue time of 0-1000ms
         Time.now - rand + @request_body_wait
       end
     end
 
-    def ignore?
-      @config.ignore_large_requests? && @size > @config.max_request_size
-    end
+    def queue_time(now = Time.now)
+      return if started_at.nil?
 
-    def queue_time
-      if @entered_queue_at
-        queue_time = ((Time.now - @entered_queue_at) * 1000).to_i
+      queue_time = ((now - started_at) * 1000).to_i
 
-        # Subtract the time Puma spent waiting on the request body. It's irrelevant to capacity-related queue time.
-        # Without this, slow clients and large request payloads will skew queue time.
-        queue_time -= @request_body_wait
+      # Subtract the time Puma spent waiting on the request body. It's irrelevant to capacity-related queue time.
+      # Without this, slow clients and large request payloads will skew queue time.
+      queue_time -= @request_body_wait
 
-        logger.debug "Request queue_time=#{queue_time}ms body_wait=#{@request_body_wait}ms request_id=#{@id} size=#{@size}"
+      logger.debug "Request queue_time=#{queue_time}ms body_wait=#{@request_body_wait}ms request_id=#{@id} size=#{@size}"
 
-        # Safeguard against negative queue times (should not happen in practice)
-        queue_time > 0 ? queue_time : 0
-      end
+      # Safeguard against negative queue times (should not happen in practice)
+      queue_time > 0 ? queue_time : 0
     end
   end
 end

data/lib/rails_autoscale_agent/store.rb

@@ -13,13 +13,19 @@ module RailsAutoscaleAgent
 
     def initialize
       @measurements = []
+      @last_pop = Time.now
     end
 
     def push(value, time = Time.now, queue_name = nil, metric = nil)
+      # If it's been two minutes since clearing out the store, stop collecting measurements.
+      # There could be an issue with the reporter, and continuing to collect will consume linear memory.
+      return if @last_pop && @last_pop < Time.now - 120
+
       @measurements << Measurement.new(time, value, queue_name, metric)
     end
 
     def pop_report
+      @last_pop = Time.now
       report = Report.new
 
       while measurement = @measurements.shift

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.9.0"
+  VERSION = "0.11.0"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -10,17 +10,11 @@ module RailsAutoscaleAgent
 
       attr_writer :queues
 
-      def queues
-        # Track the known queues so we can continue reporting on queues that don't
-        # have enqueued jobs at the time of reporting.
-        # Assume a "default" queue so we always report *something*, even when nothing
-        # is enqueued.
-        @queues ||= Set.new(['default'])
-      end
-
       def enabled?
         if defined?(::Delayed::Job) && defined?(::Delayed::Backend::ActiveRecord)
-          logger.info "DelayedJob enabled (#{::ActiveRecord::Base.default_timezone})"
+          log_msg = String.new("DelayedJob enabled (#{::ActiveRecord::Base.default_timezone})")
+          log_msg << " with long-running job support" if track_long_running_jobs?
+          logger.info log_msg
           true
         end
       end
@@ -36,8 +30,29 @@ module RailsAutoscaleAgent
           GROUP BY queue
         SQL
 
-        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-        self.queues |= run_at_by_queue.keys
+        run_at_by_queue = Hash[select_rows(sql)]
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if run_at_by_queue.size > Config.instance.max_queues
+          logger.warn "Skipping DelayedJob metrics - #{run_at_by_queue.size} queues exceeds the #{Config.instance.max_queues} queue limit"
+          return
+        end
+
+        self.queues = queues | run_at_by_queue.keys
+
+        if track_long_running_jobs?
+          sql = <<~SQL
+            SELECT COALESCE(queue, 'default'), count(*)
+            FROM delayed_jobs
+            WHERE locked_at IS NOT NULL
+            AND locked_by IS NOT NULL
+            AND failed_at IS NULL
+            GROUP BY 1
+          SQL
+
+          busy_count_by_queue = Hash[select_rows(sql)]
+          self.queues = queues | busy_count_by_queue.keys
+        end
 
         queues.each do |queue|
           run_at = run_at_by_queue[queue]
@@ -47,11 +62,36 @@ module RailsAutoscaleAgent
           latency_ms = 0 if latency_ms < 0
 
           store.push latency_ms, t, queue
-          log_msg << "dj.#{queue}=#{latency_ms} "
+          log_msg << "dj-qt.#{queue}=#{latency_ms} "
+
+          if track_long_running_jobs?
+            busy_count = busy_count_by_queue[queue] || 0
+            store.push busy_count, Time.now, queue, :busy
+            log_msg << "dj-busy.#{queue}=#{busy_count} "
+          end
         end
 
         logger.debug log_msg unless log_msg.empty?
       end
+
+      private
+
+      def queues
+        # Track the known queues so we can continue reporting on queues that don't
+        # have enqueued jobs at the time of reporting.
+        # Assume a "default" queue so we always report *something*, even when nothing
+        # is enqueued.
+        @queues ||= Set.new(['default'])
+      end
+
+      def track_long_running_jobs?
+        Config.instance.track_long_running_jobs
+      end
+
+      def select_rows(sql)
+        # This ensures the agent doesn't hold onto a DB connection any longer than necessary
+        ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
+      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -37,7 +37,14 @@ module RailsAutoscaleAgent
           GROUP BY 1
         SQL
 
-        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+        run_at_by_queue = Hash[select_rows(sql)]
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if run_at_by_queue.size > Config.instance.max_queues
+          logger.warn "Skipping Que metrics - #{run_at_by_queue.size} queues exceeds the #{Config.instance.max_queues} queue limit"
+          return
+        end
+
         self.queues |= run_at_by_queue.keys
 
         queues.each do |queue|
@@ -52,6 +59,13 @@ module RailsAutoscaleAgent
 
         logger.debug log_msg unless log_msg.empty?
       end
+
+      private
+
+      def select_rows(sql)
+        # This ensures the agent doesn't hold onto a DB connection any longer than necessary
+        ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
+      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/resque.rb

@@ -24,10 +24,17 @@ module RailsAutoscaleAgent
 
       def collect!(store)
         log_msg = String.new
+        current_queues = ::Resque.queues
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if current_queues.size > Config.instance.max_queues
+          logger.warn "Skipping Resque metrics - #{current_queues.size} queues exceeds the #{Config.instance.max_queues} queue limit"
+          return
+        end
 
         # Ensure we continue to collect metrics for known queue names, even when nothing is
         # enqueued at the time. Without this, it will appears that the agent is no longer reporting.
-        self.queues |= ::Resque.queues
+        self.queues |= current_queues
 
         queues.each do |queue|
           next if queue.nil? || queue.empty?

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -8,15 +8,15 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
-      attr_writer :known_queue_names
-
-      def known_queue_names
-        @known_queue_names ||= ['default']
-      end
+      attr_writer :queues
 
       def enabled?
         require 'sidekiq/api'
-        logger.info "Sidekiq enabled"
+
+        log_msg = String.new("Sidekiq enabled")
+        log_msg << " with long-running job support" if track_long_running_jobs?
+        logger.info log_msg
+
         true
       rescue LoadError
         false
@@ -28,23 +28,53 @@ module RailsAutoscaleAgent
           obj[queue.name] = queue
         end
 
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if queues_by_name.size > Config.instance.max_queues
+          logger.warn "Skipping Sidekiq metrics - #{queues_by_name.size} queues exceeds the #{Config.instance.max_queues} queue limit"
+          return
+        end
+
         # Ensure we continue to collect metrics for known queue names, even when nothing is
-        # enqueued at the time. Without this, it will appears that the agent is no longer reporting.
-        known_queue_names.each do |queue_name|
+        # enqueued at the time. Without this, it will appear that the agent is no longer reporting.
+        queues.each do |queue_name|
           queues_by_name[queue_name] ||= ::Sidekiq::Queue.new(queue_name)
         end
-        self.known_queue_names = queues_by_name.keys
+        self.queues = queues_by_name.keys
+
+        if track_long_running_jobs?
+          busy_counts = Hash.new { |h,k| h[k] = 0}
+          ::Sidekiq::Workers.new.each do |pid, tid, work|
+            busy_counts[work.dig('payload', 'queue')] += 1
+          end
+        end
 
         queues_by_name.each do |queue_name, queue|
           latency_ms = (queue.latency * 1000).ceil
           depth = queue.size
+
           store.push latency_ms, Time.now, queue.name, :qt
           store.push depth, Time.now, queue.name, :qd
           log_msg << "sidekiq-qt.#{queue.name}=#{latency_ms} sidekiq-qd.#{queue.name}=#{depth} "
+
+          if track_long_running_jobs?
+            busy_count = busy_counts[queue.name]
+            store.push busy_count, Time.now, queue.name, :busy
+            log_msg << "sidekiq-busy.#{queue.name}=#{busy_count} "
+          end
         end
 
         logger.debug log_msg
       end
+
+      private
+
+      def queues
+        @queues ||= ['default']
+      end
+
+      def track_long_running_jobs?
+        Config.instance.track_long_running_jobs
+      end
     end
   end
 end

data/rails_autoscale_agent.gemspec

@@ -9,10 +9,20 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adam McCrea"]
   spec.email         = ["adam@adamlogic.com"]
 
-  spec.summary       = "This gem works with the Rails Autoscale Heroku add-on to automatically scale your web dynos."
-  spec.homepage      = "https://github.com/adamlogic/rails_autoscale_agent"
+  spec.summary       = "This gem works with the Rails Autoscale Heroku add-on to automatically scale your web and worker dynos."
+  spec.homepage      = "https://railsautoscale.com"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.5.0'
+
+  spec.metadata = {
+    "homepage_uri" => "https://railsautoscale.com",
+    "bug_tracker_uri" => "https://github.com/adamlogic/rails_autoscale_agent/issues",
+    "documentation_uri" => "https://railsautoscale.com/docs",
+    "changelog_uri" => "https://github.com/adamlogic/rails_autoscale_agent/blob/master/CHANGELOG.md",
+    "source_code_uri" => "https://github.com/adamlogic/rails_autoscale_agent",
+  }
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Adam McCrea
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-13 00:00:00.000000000 Z
+date: 2022-01-05 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email:
 - adam@adamlogic.com
 executables: []
@@ -19,9 +19,9 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".ruby-version"
 - ".travis.yml"
 - ".vscode/tasks.json"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -48,11 +48,16 @@ files:
 - lib/rails_autoscale_agent/worker_adapters/sidekiq.rb
 - log/.gitkeep
 - rails_autoscale_agent.gemspec
-homepage: https://github.com/adamlogic/rails_autoscale_agent
+homepage: https://railsautoscale.com
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://railsautoscale.com
+  bug_tracker_uri: https://github.com/adamlogic/rails_autoscale_agent/issues
+  documentation_uri: https://railsautoscale.com/docs
+  changelog_uri: https://github.com/adamlogic/rails_autoscale_agent/blob/master/CHANGELOG.md
+  source_code_uri: https://github.com/adamlogic/rails_autoscale_agent
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -60,16 +65,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: This gem works with the Rails Autoscale Heroku add-on to automatically scale
-  your web dynos.
+  your web and worker dynos.
 test_files: []

data/.ruby-version

@@ -1 +0,0 @@
-2.6.5