rails_autoscale_agent 0.9.0.beta.2 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 998351ab2e804fc443d63825d3a71255d6539f225ef791a4819391d735d29e51
-  data.tar.gz: 83df02c312d3fd2a1504c782b699c6ac5e26444145ff885ecff808ebf2a9cfc3
+  metadata.gz: ed5e9567b63e205a9477e80cb3b1cb5d0d198f60288957c8560eb988f0ab5322
+  data.tar.gz: 0ff58213c0347a71218c668fbcea5f3bae8e6e24701b7c9ce3317d84dfce8ef7
 SHA512:
-  metadata.gz: 6a34893873a4fc6b926e93d89002b3527f8adce615f97d1694753995df89666cfea3c37f55d435011edbe3e6c39928f835d7594c79cd310feb2267bfc1bbb1fa
-  data.tar.gz: 00b6943d706c93e1d93dc01259d6982618083b0d7359b212eb0187c26edac6f979a11c0c7a387671d60f80a6eb1ef13e74f5182b9f92b814dc01cd9d0f8c5b6b
+  metadata.gz: 3da81efface83a93ac9c3676b7edbcaf431911cfe428c054a39203663048bc7a320ebd6881606d7b18e563c145d8fcfc1cd6d3acf945bff82525195796f1be96
+  data.tar.gz: a4012e4ab5876cddb28bfea34a2b71d6239c34dc5b167cb54616e5635a67d36915245c2b96bc536256ee7109640db6a035d0b5b99bfa20fab76a88900aba6f3f

data/.vscode/tasks.json

@@ -6,7 +6,8 @@
     {
       "label": "test: all",
       "group": "test",
-      "command": "rspec",
+      "command": "bundle",
+      "args": ["exec", "rspec"],
       "runOptions": {
         "reevaluateOnRerun": false
       },
@@ -31,8 +32,8 @@
     {
       "label": "test: file",
       "group": "test",
-      "command": "rspec",
-      "args": ["${relativeFile}"],
+      "command": "bundle",
+      "args": ["exec", "rspec", "${relativeFile}"],
       "runOptions": {
         "reevaluateOnRerun": false
       },
@@ -57,8 +58,8 @@
     {
       "label": "test: line",
       "group": "test",
-      "command": "rspec",
-      "args": ["${relativeFile}:${lineNumber}"],
+      "command": "bundle",
+      "args": ["exec", "rspec", "${relativeFile}:${lineNumber}"],
       "runOptions": {
         "reevaluateOnRerun": false
       },

data/CHANGELOG.md

@@ -0,0 +1,109 @@
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.10.1...master)
+
+_There are no currently unreleased changes._
+
+## [0.10.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.1...v0.10.1) - 2021-01-03
+
+### Added
+
+- Add support for [long-running jobs](https://railsautoscale.com/docs/long-running-jobs/) in Sidekiq and Delayed Job.
+- Handle x-request-start measured in seconds (instead of milliseconds) to support nginx buildpack ([cd092f3](https://github.com/adamlogic/rails_autoscale_agent/commit/cd092f38718abf5ffaea866bcae7831d4c910ffd))
+- Override worker adapter config via env var ([75dd06b](https://github.com/adamlogic/rails_autoscale_agent/commit/75dd06b2a7ff4eeab829eec24d503dc067c8fe32))
+
+### Changed
+
+- Require Ruby 2.5 or newer. ([b033050](https://github.com/adamlogic/rails_autoscale_agent/commit/b033050b7f9d4d7f1e50dbd780cf0e1822249268))
+- Only report worker metrics from web.1 to avoid redundant data. ([d5d5fa8](https://github.com/adamlogic/rails_autoscale_agent/commit/d5d5fa87fb4d7d046832a64edde9ed0c3a6ec75f))
+- Don't collect worker metrics for an unreasonable number of queues. ([a9358af](https://github.com/adamlogic/rails_autoscale_agent/commit/a9358af74a29a941d1f1d60a0222077dafd5ce08))
+
+### Fixed
+
+- Avoid holding onto database connections (DJ & Que only). ([3919ca5](https://github.com/adamlogic/rails_autoscale_agent/commit/3919ca54420cafa82abf9f8cd251569f9637482b))
+- Better error handling for worker adapters. ([190786e](https://github.com/adamlogic/rails_autoscale_agent/commit/190786e4a910d41e394a3129aac1d23b594dbd9b))
+- Don't collect metrics of the reporter isn't running. Avoids memory bloat. ([247c322](https://github.com/adamlogic/rails_autoscale_agent/commit/247c322cffc625a8c6b2395080a048ffb94e7f3b))
+
+## [0.10.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.1...v0.10.0) - 2021-01-03 [YANKED]
+
+_I released the wrong branch 🤦‍♂️_
+
+## [0.9.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.0...v0.9.1) - 2020-07-29
+
+### Fixed
+
+- Fix a bug in error handling. ([3018542](https://github.com/adamlogic/rails_autoscale_agent/commit/3018542cd046fc4e1bd6e7da86e72a6aa2d50a8f))
+- Remove unintentional Rails dependency.
+
+## [0.9.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.3...v0.9.0) - 2020-07-12
+
+### Added
+
+- Add support for Resque workers.
+- Add dev mode for working on the agent gem itself. ([47e3fca](https://github.com/adamlogic/rails_autoscale_agent/commit/47e3fca5b788f48567a345d9cab3a26b9cd87693))
+- Report agent exceptions to Rails Autoscale.
+
+### Changed
+
+- Adjust queue time metric to exclude time waiting for large request bodies. ([#25](https://github.com/adamlogic/rails_autoscale_agent/pull/25))
+- Que and DJ jobs without a queue name will be included in the "default" queue metrics.
+
+### Fixed
+
+- Multiple fixes to the Delayed Job SQL query.
+
+## [0.8.3](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.2...v0.8.3) - 2020-05-26
+
+### Fixed
+
+- Ignored failed job in Delayed Job adapter. ([fa72fc2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.2...v0.8.3))
+
+## [0.8.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.1...v0.8.2) - 2020-05-22
+
+### Fixed
+
+- Ignore worker metrics from unnamed queues (DJ & Que only). These metrics were being lumped with web metrics. ([#21](https://github.com/adamlogic/rails_autoscale_agent/pull/21))
+
+## [0.8.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.0...v0.8.1) - 2020-05-04
+
+### Fixed
+
+- Ignore failed jobs in Que adapter. ([#18](https://github.com/adamlogic/rails_autoscale_agent/pull/18))
+
+## [0.8.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.7.0...v0.8.0) - 2020-03-21
+
+### Added
+
+- Add support for Delayed Job ([#14](https://github.com/adamlogic/rails_autoscale_agent/pull/14))
+- Add support for Que ([#15](https://github.com/adamlogic/rails_autoscale_agent/pull/15))
+
+## [0.7.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.3...v0.7.0) - 2019-12-04
+
+### Added
+
+- Make worker adapters configurable. ([012d937](https://github.com/adamlogic/rails_autoscale_agent/commit/012d9379296763f5e42df95f05b066fe82ab0051))
+
+## [0.6.3](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.2...v0.6.3) - 2019-06-25
+
+### Fixed
+
+- Fix issues with logging.
+
+## [0.6.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.1...v0.6.2) - 2019-06-22
+
+### Fixed
+
+- Fix issues with logging.
+
+## [0.6.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.0...v0.6.1) - 2019-05-06
+
+### Fixed
+
+- Don't assume Sidekiq is present.
+
+## [0.6.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.4.1...v0.6.0) - 2019-05-03
+
+### Added
+
+- Add support for autoscaling Sidekiq.

data/Gemfile

@@ -1,4 +1,16 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in rails_autoscale_agent.gemspec
 gemspec
+
+gem "rake", ">= 12.3.3"
+gem "rspec", ">= 3.0"
+gem "vcr", ">= 3.0"
+gem "webmock"
+gem "pry-byebug"
+gem "sidekiq", ">= 5.0"
+gem "delayed_job"
+gem "delayed_job_active_record"
+gem "que"
+gem "resque"
+gem "activesupport"
+gem "sqlite3", platforms: :ruby

data/README.md

@@ -22,17 +22,7 @@ The agent will only communicate with Rails Autoscale if a `RAILS_AUTOSCALE_URL`
 
 ## Non-Rails Rack apps
 
-You'll need to insert the `RailsAutoscaleAgent::Middleware` manually. Insert it before `Rack::Runtime` to ensure accuracy of request queue timings.
-
-## Changing the logger
-
-The Rails logger is used by default.
-If you wish to use a different logger you can set it on the configuration object:
-
-```ruby
-# config/initializers/rails_autoscale_agent.rb
-RailsAutoscaleAgent::Config.instance.logger = MyLogger.new
-```
+You'll need to `require 'rails_autoscale_agent/middleware'` and insert the `RailsAutoscaleAgent::Middleware` manually. Insert it before `Rack::Runtime` to ensure accuracy of request queue timings.
 
 ## What data is collected?
 
@@ -47,6 +37,32 @@ The middleware agent runs in its own thread so your web requests are not impacte
 
 Rails Autoscale aggregates and stores this information to power the autoscaling algorithm and dashboard visualizations.
 
+## Configuration
+
+Most Rails Autoscale configurations are handled via the settings page on your Rails Autoscale dashboard, but there a few ways you can directly change the behavior of the agent via environment variables:
+
+- `RAILS_AUTOSCALE_DEBUG` - Enables debug logging. See more in the [logging](#logging) section below.
+- `RAILS_AUTOSCALE_WORKER_ADAPTER` - Overrides the available worker adapters. See more in the [worker adapters](#worker_adapters) section below.
+- `RAILS_AUTOSCALE_LONG_JOBS` - Enables reporting for active workers. See [Handling Long-Running Background Jobs](https://railsautoscale.com/docs/long-running-jobs/) in the Rails Autoscale docs for more.
+
+## Worker adapters
+
+Rails Autoscale supports autoscaling worker dynos. Out of the box, four job backends are supported: Sidekiq, Resque, Delayed Job, and Que. The agent will automatically enable the appropriate worker adapter based on what you have installed in your app.
+
+In some scenarios you might want to override this behavior. Let's say you have both Sidekiq and Resque installed 🤷‍♂️, but you only want Rails Autoscale to collect metrics for Sidekiq. Here's how you'd override that:
+
+```
+heroku config:add RAILS_AUTOSCALE_WORKER_ADAPTER=sidekiq
+```
+
+You can also disable collection of worker metrics altogether:
+
+```
+heroku config:add RAILS_AUTOSCALE_WORKER_ADAPTER=""
+```
+
+It's also possible to write a custom worker adapter. See [these docs](https://railsautoscale.com/docs/custom-worker-adapter/) for details.
+
 ## Troubleshooting
 
 Once installed, you should see something like this in your development log:
@@ -59,17 +75,39 @@ In production, run `heroku logs -t | grep RailsAutoscale`, and you should see so
 
 If you don't see either of these, try running `bundle` again and restarting your Rails application.
 
-You can see more detailed (debug) logging by setting the `RAILS_AUTOSCALE_DEBUG` env var on your Heroku app:
+You can see more detailed (debug) logging by setting `RAILS_AUTOSCALE_DEBUG` on your Heroku app:
 
 ```
 heroku config:add RAILS_AUTOSCALE_DEBUG=true
 ```
 
-Debug logs are silenced by default because Rails apps default to a DEBUG log level in production,
-and these can get very noisy with this gem.
+See more in the [logging](#logging) section below.
 
 Reach out to help@railsautoscale.com if you run into any other problems.
 
+## Logging
+
+The Rails logger is used by default.
+If you wish to use a different logger you can set it on the configuration object:
+
+```ruby
+# config/initializers/rails_autoscale_agent.rb
+RailsAutoscaleAgent::Config.instance.logger = MyLogger.new
+```
+
+Debug logs are silenced by default because Rails apps default to a DEBUG log level in production, and this gem has _very_ chatty debug logs. If you want to see the debug logs, set `RAILS_AUTOSCALE_DEBUG` on your Heroku app:
+
+```
+heroku config:add RAILS_AUTOSCALE_DEBUG=true
+```
+
+If you find the gem too chatty even without this, you can quiet it down further:
+
+```ruby
+# config/initializers/rails_autoscale_agent.rb
+RailsAutoscaleAgent::Config.instance.quiet = true
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/rails_autoscale_agent/autoscale_api.rb

@@ -59,7 +59,7 @@ module RailsAutoscaleAgent
 
       case response.code.to_i
       when 200...300 then SuccessResponse.new(response.body)
-      else FailureResponse.new(response.message)
+      else FailureResponse.new([response.code, response.message].join(' - '))
       end
     end
 

data/lib/rails_autoscale_agent/config.rb

@@ -4,36 +4,34 @@ require 'singleton'
 
 module RailsAutoscaleAgent
   class Config
+    DEFAULT_WORKER_ADAPTERS = 'sidekiq,delayed_job,que,resque'
+
     include Singleton
 
     attr_accessor :report_interval, :logger, :api_base_url, :max_request_size,
-                  :dyno, :pid, :addon_name, :worker_adapters, :dev_mode
+                  :dyno, :addon_name, :worker_adapters, :dev_mode, :debug, :quiet,
+                  :track_long_running_jobs,
+
+                  # legacy configs, no longer used
+                  :sidekiq_latency_for_active_jobs, :latency_for_active_jobs
 
     def initialize
-      require 'rails_autoscale_agent/worker_adapters/sidekiq'
-      require 'rails_autoscale_agent/worker_adapters/delayed_job'
-      require 'rails_autoscale_agent/worker_adapters/que'
-      require 'rails_autoscale_agent/worker_adapters/resque'
-      @worker_adapters = [
-        WorkerAdapters::Sidekiq.instance,
-        WorkerAdapters::DelayedJob.instance,
-        WorkerAdapters::Que.instance,
-        WorkerAdapters::Resque.instance,
-      ]
+      @worker_adapters = prepare_worker_adapters
 
       # Allow the add-on name to be configured - needed for testing
       @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
       @api_base_url = ENV["#{@addon_name}_URL"]
       @dev_mode = ENV['RAILS_AUTOSCALE_DEV'] == 'true'
-      @pid = Process.pid
+      @debug = dev_mode? || ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
+      @track_long_running_jobs = ENV['RAILS_AUTOSCALE_LONG_JOBS'] == 'true'
       @max_request_size = 100_000 # ignore request payloads over 100k since they skew the queue times
       @report_interval = 10 # this default will be overwritten during Reporter#register!
       @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
-      @dyno = @dev_mode ? 'dev.1' : ENV['DYNO']
+      @dyno = dev_mode? ? 'dev.1' : ENV['DYNO']
     end
 
     def to_s
-      "#{@dyno}##{@pid}"
+      "#{@dyno}##{Process.pid}"
     end
 
     def ignore_large_requests?
@@ -41,5 +39,18 @@ module RailsAutoscaleAgent
     end
 
     alias_method :dev_mode?, :dev_mode
+    alias_method :debug?, :debug
+    alias_method :quiet?, :quiet
+
+    private
+
+    def prepare_worker_adapters
+      adapter_names = (ENV['RAILS_AUTOSCALE_WORKER_ADAPTER'] || DEFAULT_WORKER_ADAPTERS).split(',')
+      adapter_names.map do |adapter_name|
+        require "rails_autoscale_agent/worker_adapters/#{adapter_name}"
+        adapter_constant_name = adapter_name.capitalize.gsub(/(?:_)(.)/i) { $1.upcase }
+        WorkerAdapters.const_get(adapter_constant_name).instance
+      end
+    end
   end
 end

data/lib/rails_autoscale_agent/logger.rb

@@ -13,26 +13,33 @@ module RailsAutoscaleAgent
   class LoggerProxy < Struct.new(:logger)
     TAG = '[RailsAutoscale]'
 
-    %w[info warn error].each do |name|
-      define_method name do |msg|
-        logger.send name, tag(msg)
-      end
+    def error(msg)
+      logger.error tag(msg)
+    end
+
+    def warn(msg)
+      logger.warn tag(msg)
+    end
+
+    def info(msg)
+      logger.info tag(msg) unless Config.instance.quiet?
     end
 
     def debug(msg)
       # Silence debug logs by default to avoiding being overly chatty (Rails logger defaults
-      # to DEBUG level in production).
-      # This uses a separate logger so that RAILS_AUTOSCALE_DEBUG
-      # shows debug logs regardless of Rails log level.
-      debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true' || Config.instance.dev_mode?
+      # to DEBUG level in production). Setting RAILS_AUTOSCALE_DEBUG=true enables debug logs,
+      # even if the underlying logger severity level is INFO.
+      if Config.instance.debug?
+        if logger.respond_to?(:debug?) && logger.debug?
+          logger.debug tag(msg)
+        elsif logger.respond_to?(:info?) && logger.info?
+          logger.info tag("[DEBUG] #{msg}")
+        end
+      end
     end
 
     private
 
-    def debug_logger
-      @debug_loggers ||= ::Logger.new(STDOUT)
-    end
-
     def tag(msg)
       "#{TAG} #{msg}"
     end

data/lib/rails_autoscale_agent/registration.rb

@@ -8,7 +8,7 @@ module RailsAutoscaleAgent
     def to_params
       {
         dyno: config.dyno,
-        pid: config.pid,
+        pid: Process.pid,
         ruby_version: RUBY_VERSION,
         rails_version: defined?(Rails) && Rails.version,
         gem_version: VERSION,

data/lib/rails_autoscale_agent/report.rb

@@ -12,7 +12,7 @@ module RailsAutoscaleAgent
     def to_params(config)
       {
         dyno: config.dyno,
-        pid: config.pid,
+        pid: Process.pid,
       }
     end
 

data/lib/rails_autoscale_agent/reporter.rb

@@ -20,6 +20,7 @@ module RailsAutoscaleAgent
     def start!(config, store)
       @started = true
       @worker_adapters = config.worker_adapters.select(&:enabled?)
+      @dyno_num = config.dyno.to_s.split('.').last.to_i
 
       if !config.api_base_url && !config.dev_mode?
         logger.info "Reporter not started: #{config.addon_name}_URL is not set"
@@ -34,15 +35,14 @@ module RailsAutoscaleAgent
           multiplier = 1 - (rand / 4) # between 0.75 and 1.0
           sleep config.report_interval * multiplier
 
-          begin
-            @worker_adapters.map { |a| a.collect!(store) }
-            report!(config, store)
-          rescue => ex
-            # Exceptions in threads other than the main thread will fail silently
-            # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
-            logger.error "Reporter error: #{ex.inspect}"
-            AutoscaleApi.new(config.api_base_url).report_exception!(ex)
+          # It's redundant to report worker metrics from every web dyno, so only report from web.1
+          if @dyno_num == 1
+            @worker_adapters.map do |adapter|
+              report_exceptions(config) { adapter.collect!(store) }
+            end
           end
+
+          report_exceptions(config) { report!(config, store) }
         end
       end
     end
@@ -51,6 +51,8 @@ module RailsAutoscaleAgent
       @started
     end
 
+    private
+
     def report!(config, store)
       report = store.pop_report
 
@@ -86,5 +88,20 @@ module RailsAutoscaleAgent
         logger.error "Reporter failed to register: #{result.failure_message}"
       end
     end
+
+    def report_exceptions(config)
+      begin
+        yield
+      rescue => ex
+        # Exceptions in threads other than the main thread will fail silently
+        # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
+        logger.error "Reporter error: #{ex.inspect}"
+        AutoscaleApi.new(config).report_exception!(ex)
+      end
+    rescue => ex
+      # An exception was encountered while trying to report the original exception.
+      # Swallow the error so the reporter continues to report.
+      logger.error "Exception reporting error: #{ex.inspect}"
+    end
   end
 end

data/lib/rails_autoscale_agent/request.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'rails_autoscale_agent/logger'
+
 module RailsAutoscaleAgent
   class Request
     include Logger
@@ -9,32 +11,39 @@ module RailsAutoscaleAgent
       @id = env['HTTP_X_REQUEST_ID']
       @size = env['rack.input'].respond_to?(:size) ? env['rack.input'].size : 0
       @request_body_wait = env['puma.request_body_wait'].to_i
+      @request_start_header = env['HTTP_X_REQUEST_START']
+    end
+
+    def ignore?
+      @config.ignore_large_requests? && @size > @config.max_request_size
+    end
 
-      @entered_queue_at = if unix_millis = env['HTTP_X_REQUEST_START']
-        Time.at(unix_millis.to_f / 1000)
-      elsif config.dev_mode?
+    def started_at
+      if @request_start_header
+        # Heroku sets the header as an integer, measured in milliseconds.
+        # If nginx is involved, it might be in seconds with fractional milliseconds,
+        # and it might be preceeded by "t=". We can all cases by removing non-digits
+        # and treating as milliseconds.
+        Time.at(@request_start_header.gsub(/\D/, '').to_i / 1000.0)
+      elsif @config.dev_mode?
         # In dev mode, fake a queue time of 0-1000ms
         Time.now - rand + @request_body_wait
       end
     end
 
-    def ignore?
-      @config.ignore_large_requests? && @size > @config.max_request_size
-    end
+    def queue_time(now = Time.now)
+      return if started_at.nil?
 
-    def queue_time
-      if @entered_queue_at
-        queue_time = ((Time.now - @entered_queue_at) * 1000).to_i
+      queue_time = ((now - started_at) * 1000).to_i
 
-        # Subtract the time Puma spent waiting on the request body. It's irrelevant to capacity-related queue time.
-        # Without this, slow clients and large request payloads will skew queue time.
-        queue_time -= @request_body_wait
+      # Subtract the time Puma spent waiting on the request body. It's irrelevant to capacity-related queue time.
+      # Without this, slow clients and large request payloads will skew queue time.
+      queue_time -= @request_body_wait
 
-        logger.debug "Request queue_time=#{queue_time}ms body_wait=#{@request_body_wait}ms request_id=#{@id} size=#{@size}"
+      logger.debug "Request queue_time=#{queue_time}ms body_wait=#{@request_body_wait}ms request_id=#{@id} size=#{@size}"
 
-        # Safeguard against negative queue times (should not happen in practice)
-        queue_time > 0 ? queue_time : 0
-      end
+      # Safeguard against negative queue times (should not happen in practice)
+      queue_time > 0 ? queue_time : 0
     end
   end
 end

data/lib/rails_autoscale_agent/store.rb

@@ -16,10 +16,15 @@ module RailsAutoscaleAgent
     end
 
     def push(value, time = Time.now, queue_name = nil, metric = nil)
+      # If it's been two minutes since clearing out the store, stop collecting measurements.
+      # There could be an issue with the reporter, and continuing to collect will consume linear memory.
+      return if @last_pop && @last_pop < Time.now - 120
+
       @measurements << Measurement.new(time, value, queue_name, metric)
     end
 
     def pop_report
+      @last_pop = Time.now
       report = Report.new
 
       while measurement = @measurements.shift

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.9.0.beta.2"
+  VERSION = "0.10.1"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -8,38 +8,67 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
-      class << self
-        attr_accessor :queues
-      end
-
-      def initialize
-        # Track the known queues so we can continue reporting on queues that don't
-        # currently have enqueued jobs.
-        self.class.queues = Set.new
-
-        install if enabled?
-      end
+      attr_writer :queues
 
       def enabled?
-        defined? ::Delayed
+        if defined?(::Delayed::Job) && defined?(::Delayed::Backend::ActiveRecord)
+          log_msg = String.new("DelayedJob enabled (#{::ActiveRecord::Base.default_timezone})")
+          log_msg << " with long-running job support" if track_long_running_jobs?
+          logger.info log_msg
+          true
+        end
       end
 
       def collect!(store)
         log_msg = String.new
-        t = Time.now
+        t = Time.now.utc
+        sql = <<~SQL
+          SELECT COALESCE(queue, 'default'), min(run_at)
+          FROM delayed_jobs
+          WHERE locked_at IS NULL
+          AND failed_at IS NULL
+          GROUP BY queue
+        SQL
+
+        run_at_by_queue = Hash[select_rows(sql)]
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if run_at_by_queue.size > 50
+          logger.debug "Skipping DelayedJob metrics - #{run_at_by_queue.size} queues"
+          return
+        end
+
+        self.queues = queues | run_at_by_queue.keys
 
-        # Ignore failed jobs (they skew latency measurement due to the original run_at)
-        sql = 'SELECT queue, min(run_at) FROM delayed_jobs WHERE attempts = 0 GROUP BY queue'
-        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-        queues = self.class.queues | run_at_by_queue.keys
+        if track_long_running_jobs?
+          sql = <<~SQL
+            SELECT COALESCE(queue, 'default'), count(*)
+            FROM delayed_jobs
+            WHERE locked_at IS NOT NULL
+            AND locked_by IS NOT NULL
+            AND failed_at IS NULL
+            GROUP BY 1
+          SQL
+
+          busy_count_by_queue = Hash[select_rows(sql)]
+          self.queues = queues | busy_count_by_queue.keys
+        end
 
         queues.each do |queue|
-          next if queue.nil? || queue.empty?
           run_at = run_at_by_queue[queue]
-          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          # DateTime.parse assumes a UTC string
+          run_at = DateTime.parse(run_at) if run_at.is_a?(String)
           latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          latency_ms = 0 if latency_ms < 0
+
           store.push latency_ms, t, queue
-          log_msg << "dj.#{queue}=#{latency_ms} "
+          log_msg << "dj-qt.#{queue}=#{latency_ms} "
+
+          if track_long_running_jobs?
+            busy_count = busy_count_by_queue[queue] || 0
+            store.push busy_count, Time.now, queue, :busy
+            log_msg << "dj-busy.#{queue}=#{busy_count} "
+          end
         end
 
         logger.debug log_msg unless log_msg.empty?
@@ -47,19 +76,21 @@ module RailsAutoscaleAgent
 
       private
 
-      def install
-        plugin = Class.new(Delayed::Plugin) do
-          require 'delayed_job'
+      def queues
+        # Track the known queues so we can continue reporting on queues that don't
+        # have enqueued jobs at the time of reporting.
+        # Assume a "default" queue so we always report *something*, even when nothing
+        # is enqueued.
+        @queues ||= Set.new(['default'])
+      end
 
-          callbacks do |lifecycle|
-            lifecycle.before(:enqueue) do |job, &block|
-              queue = job.queue || 'default'
-              WorkerAdapters::DelayedJob.queues.add queue
-            end
-          end
-        end
+      def track_long_running_jobs?
+        Config.instance.track_long_running_jobs
+      end
 
-        Delayed::Worker.plugins << plugin
+      def select_rows(sql)
+        # This ensures the agent doesn't hold onto a DB connection any longer than necessary
+        ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
       end
     end
   end

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'rails_autoscale_agent/logger'
-require 'time'
 
 module RailsAutoscaleAgent
   module WorkerAdapters
@@ -9,40 +8,64 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
-      DEFAULT_QUEUES = ['default']
+      attr_writer :queues
 
-      class << self
-        attr_accessor :queues
-      end
-
-      def initialize
-        self.class.queues = DEFAULT_QUEUES
+      def queues
+        # Track the known queues so we can continue reporting on queues that don't
+        # have enqueued jobs at the time of reporting.
+        # Assume a "default" queue so we always report *something*, even when nothing
+        # is enqueued.
+        @queues ||= Set.new(['default'])
       end
 
       def enabled?
-        defined? ::Que
+        if defined?(::Que)
+          logger.info "Que enabled (#{::ActiveRecord::Base.default_timezone})"
+          true
+        end
       end
 
       def collect!(store)
         log_msg = String.new
-        t = Time.now
+        t = Time.now.utc
+        sql = <<~SQL
+          SELECT queue, min(run_at)
+          FROM que_jobs
+          WHERE finished_at IS NULL
+          AND expired_at IS NULL
+          AND error_count = 0
+          GROUP BY 1
+        SQL
+
+        run_at_by_queue = Hash[select_rows(sql)]
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if run_at_by_queue.size > 50
+          logger.debug "Skipping Que metrics - #{run_at_by_queue.size} queues"
+          return
+        end
 
-        # Ignore failed jobs (they skew latency measurement due to the original run_at)
-        sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
-        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-        self.class.queues |= run_at_by_queue.keys
+        self.queues |= run_at_by_queue.keys
 
-        self.class.queues.each do |queue|
-          next if queue.nil? || queue.empty?
+        queues.each do |queue|
           run_at = run_at_by_queue[queue]
-          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          run_at = DateTime.parse(run_at) if run_at.is_a?(String)
           latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          latency_ms = 0 if latency_ms < 0
+
           store.push latency_ms, t, queue
           log_msg << "que.#{queue}=#{latency_ms} "
         end
 
         logger.debug log_msg unless log_msg.empty?
       end
+
+      private
+
+      def select_rows(sql)
+        # This ensures the agent doesn't hold onto a DB connection any longer than necessary
+        ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
+      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/resque.rb

@@ -8,8 +8,15 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
+      attr_writer :queues
+
+      def queues
+        @queues ||= ['default']
+      end
+
       def enabled?
         require 'resque'
+        logger.info "Resque enabled"
         true
       rescue LoadError
         false
@@ -17,8 +24,19 @@ module RailsAutoscaleAgent
 
       def collect!(store)
         log_msg = String.new
+        current_queues = ::Resque.queues
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if current_queues.size > 50
+          logger.debug "Skipping Resque metrics - #{current_queues.size} queues"
+          return
+        end
+
+        # Ensure we continue to collect metrics for known queue names, even when nothing is
+        # enqueued at the time. Without this, it will appears that the agent is no longer reporting.
+        self.queues |= current_queues
 
-        ::Resque.queues.each do |queue|
+        queues.each do |queue|
           next if queue.nil? || queue.empty?
           depth = ::Resque.size(queue)
           store.push depth, Time.now, queue, :qd

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -8,8 +8,15 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
+      attr_writer :queues
+
       def enabled?
         require 'sidekiq/api'
+
+        log_msg = String.new("Sidekiq enabled")
+        log_msg << " with long-running job support" if track_long_running_jobs?
+        logger.info log_msg
+
         true
       rescue LoadError
         false
@@ -17,17 +24,57 @@ module RailsAutoscaleAgent
 
       def collect!(store)
         log_msg = String.new
+        queues_by_name = ::Sidekiq::Queue.all.each_with_object({}) do |queue, obj|
+          obj[queue.name] = queue
+        end
 
-        ::Sidekiq::Queue.all.each do |queue|
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if queues_by_name.size > 50
+          logger.debug "Skipping Sidekiq metrics - #{queues_by_name.size} queues"
+          return
+        end
+
+        # Ensure we continue to collect metrics for known queue names, even when nothing is
+        # enqueued at the time. Without this, it will appear that the agent is no longer reporting.
+        queues.each do |queue_name|
+          queues_by_name[queue_name] ||= ::Sidekiq::Queue.new(queue_name)
+        end
+        self.queues = queues_by_name.keys
+
+        if track_long_running_jobs?
+          busy_counts = Hash.new { |h,k| h[k] = 0}
+          ::Sidekiq::Workers.new.each do |pid, tid, work|
+            busy_counts[work.dig('payload', 'queue')] += 1
+          end
+        end
+
+        queues_by_name.each do |queue_name, queue|
           latency_ms = (queue.latency * 1000).ceil
           depth = queue.size
+
           store.push latency_ms, Time.now, queue.name, :qt
           store.push depth, Time.now, queue.name, :qd
           log_msg << "sidekiq-qt.#{queue.name}=#{latency_ms} sidekiq-qd.#{queue.name}=#{depth} "
+
+          if track_long_running_jobs?
+            busy_count = busy_counts[queue.name]
+            store.push busy_count, Time.now, queue.name, :busy
+            log_msg << "sidekiq-busy.#{queue.name}=#{busy_count} "
+          end
         end
 
         logger.debug log_msg
       end
+
+      private
+
+      def queues
+        @queues ||= ['default']
+      end
+
+      def track_long_running_jobs?
+        Config.instance.track_long_running_jobs
+      end
     end
   end
 end

data/rails_autoscale_agent.gemspec

@@ -16,15 +16,5 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 12.3.3"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "vcr", "~> 3.0"
-  spec.add_development_dependency "webmock"
-  spec.add_development_dependency "pry"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "sidekiq", "~> 5.0"
-  spec.add_development_dependency "delayed_job"
-  spec.add_development_dependency "que"
-  spec.add_development_dependency "activesupport"
+  spec.required_ruby_version = '~> 2.5'
 end

metadata

@@ -1,170 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.9.0.beta.2
+  version: 0.10.1
 platform: ruby
 authors:
 - Adam McCrea
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-10 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 12.3.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 12.3.3
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: delayed_job
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: que
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
+date: 2021-01-03 00:00:00.000000000 Z
+dependencies: []
+description:
 email:
 - adam@adamlogic.com
 executables: []
@@ -176,6 +22,7 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - ".vscode/tasks.json"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -206,23 +53,23 @@ homepage: https://github.com/adamlogic/rails_autoscale_agent
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - "~>"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: This gem works with the Rails Autoscale Heroku add-on to automatically scale
   your web dynos.