rails_autoscale_agent 0.7.0 → 0.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 355209571af7f1f6c5fbe38ed112a2936903afcdd9cff56b9c68531be45c8d5e
-  data.tar.gz: 9855ff92fcc103546a4fe3115942b4e94e0c5e3e626d6b14b26c1eeab0953bc8
+  metadata.gz: fd7549c359dd5f45dbd5d44113c04413685c0cabc12836c1a88559404104ea8b
+  data.tar.gz: b193f06500d2d281e22b2684364e2aad7107dcd1713f8e30d98f2cc3a814a99f
 SHA512:
-  metadata.gz: 1e76f3c52649b97c5919ce06170414e0d4f2642bf133f781ee3e54e842f77e0264c37f625e1f4c1c681ce79aec162bb7749d133a663e7d580cfc5385813f591b
-  data.tar.gz: c312bfb9654d06c25be2bda4da119e74e547fc24f0b71cd412fdeef63665154b37bf013beaae9ad1c4fb169469c0bff4cc0a601942bbe845bb739708ecc0d426
+  metadata.gz: 0ecc593e4d6a404bbeb0d082047c87511e5be55daa77169a4159f0a96c8bc1bd7c5b78e09e712ec90119e0198c615849331de9b8ff1fd48e3f2ecf631e8395a5
+  data.tar.gz: b4802270f5a2ac21f5c8fee31c200dcd2fee5b8771591a3b9c9eb21b1a716ec59e56e24e958afbbc8ea5aa49670e0bf62dc64ad8d3670f8cb4ff7126afaf6f4f

data/.vscode/tasks.json

@@ -0,0 +1,85 @@
+{
+  // See https://go.microsoft.com/fwlink/?LinkId=733558
+  // for the documentation about the tasks.json format
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "test: all",
+      "group": "test",
+      "command": "bundle",
+      "args": ["exec", "rspec"],
+      "runOptions": {
+        "reevaluateOnRerun": false
+      },
+      "problemMatcher": {
+        "owner": "ruby",
+        "fileLocation": ["relative", "${workspaceRoot}"],
+        "severity": "error",
+        "pattern": [
+          {
+            "regexp": "^\\s{7}(.*)$",
+            "message": 1
+          },
+          {
+            "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
+            "file": 1,
+            "line": 2,
+            "code": 3
+          }
+        ]
+      }
+    },
+    {
+      "label": "test: file",
+      "group": "test",
+      "command": "bundle",
+      "args": ["exec", "rspec", "${relativeFile}"],
+      "runOptions": {
+        "reevaluateOnRerun": false
+      },
+      "problemMatcher": {
+        "owner": "ruby",
+        "fileLocation": ["relative", "${workspaceRoot}"],
+        "severity": "error",
+        "pattern": [
+          {
+            "regexp": "^\\s{7}(.*)$",
+            "message": 1
+          },
+          {
+            "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
+            "file": 1,
+            "line": 2,
+            "code": 3
+          }
+        ]
+      }
+    },
+    {
+      "label": "test: line",
+      "group": "test",
+      "command": "bundle",
+      "args": ["exec", "rspec", "${relativeFile}:${lineNumber}"],
+      "runOptions": {
+        "reevaluateOnRerun": false
+      },
+      "problemMatcher": {
+        "owner": "ruby",
+        "fileLocation": ["relative", "${workspaceRoot}"],
+        "severity": "error",
+        "pattern": [
+          {
+            "regexp": "^\\s{7}(.*)$",
+            "message": 1
+          },
+          {
+            "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
+            "file": 1,
+            "line": 2,
+            "code": 3
+          }
+        ]
+      }
+    }
+  ]
+}

data/CHANGELOG.md

@@ -0,0 +1,115 @@
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.10.2...master)
+
+No currently unreleased changes.
+
+## [0.10.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.10.1...v0.10.2) - 2021-01-12
+
+### Changed
+
+- Loosen Ruby constraint to allow Ruby 3. ([#36](https://github.com/adamlogic/rails_autoscale_agent/pull/36))
+
+## [0.10.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.1...v0.10.1) - 2021-01-03
+
+### Added
+
+- Add support for [long-running jobs](https://railsautoscale.com/docs/long-running-jobs/) in Sidekiq and Delayed Job.
+- Handle x-request-start measured in seconds (instead of milliseconds) to support nginx buildpack ([cd092f3](https://github.com/adamlogic/rails_autoscale_agent/commit/cd092f38718abf5ffaea866bcae7831d4c910ffd))
+- Override worker adapter config via env var ([75dd06b](https://github.com/adamlogic/rails_autoscale_agent/commit/75dd06b2a7ff4eeab829eec24d503dc067c8fe32))
+
+### Changed
+
+- Require Ruby 2.5 or newer. ([b033050](https://github.com/adamlogic/rails_autoscale_agent/commit/b033050b7f9d4d7f1e50dbd780cf0e1822249268))
+- Only report worker metrics from web.1 to avoid redundant data. ([d5d5fa8](https://github.com/adamlogic/rails_autoscale_agent/commit/d5d5fa87fb4d7d046832a64edde9ed0c3a6ec75f))
+- Don't collect worker metrics for an unreasonable number of queues. ([a9358af](https://github.com/adamlogic/rails_autoscale_agent/commit/a9358af74a29a941d1f1d60a0222077dafd5ce08))
+
+### Fixed
+
+- Avoid holding onto database connections (DJ & Que only). ([3919ca5](https://github.com/adamlogic/rails_autoscale_agent/commit/3919ca54420cafa82abf9f8cd251569f9637482b))
+- Better error handling for worker adapters. ([190786e](https://github.com/adamlogic/rails_autoscale_agent/commit/190786e4a910d41e394a3129aac1d23b594dbd9b))
+- Don't collect metrics of the reporter isn't running. Avoids memory bloat. ([247c322](https://github.com/adamlogic/rails_autoscale_agent/commit/247c322cffc625a8c6b2395080a048ffb94e7f3b))
+
+## [0.10.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.1...v0.10.0) - 2021-01-03 [YANKED]
+
+_I released the wrong branch 🤦‍♂️_
+
+## [0.9.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.9.0...v0.9.1) - 2020-07-29
+
+### Fixed
+
+- Fix a bug in error handling. ([3018542](https://github.com/adamlogic/rails_autoscale_agent/commit/3018542cd046fc4e1bd6e7da86e72a6aa2d50a8f))
+- Remove unintentional Rails dependency.
+
+## [0.9.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.3...v0.9.0) - 2020-07-12
+
+### Added
+
+- Add support for Resque workers.
+- Add dev mode for working on the agent gem itself. ([47e3fca](https://github.com/adamlogic/rails_autoscale_agent/commit/47e3fca5b788f48567a345d9cab3a26b9cd87693))
+- Report agent exceptions to Rails Autoscale.
+
+### Changed
+
+- Adjust queue time metric to exclude time waiting for large request bodies. ([#25](https://github.com/adamlogic/rails_autoscale_agent/pull/25))
+- Que and DJ jobs without a queue name will be included in the "default" queue metrics.
+
+### Fixed
+
+- Multiple fixes to the Delayed Job SQL query.
+
+## [0.8.3](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.2...v0.8.3) - 2020-05-26
+
+### Fixed
+
+- Ignored failed job in Delayed Job adapter. ([fa72fc2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.2...v0.8.3))
+
+## [0.8.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.1...v0.8.2) - 2020-05-22
+
+### Fixed
+
+- Ignore worker metrics from unnamed queues (DJ & Que only). These metrics were being lumped with web metrics. ([#21](https://github.com/adamlogic/rails_autoscale_agent/pull/21))
+
+## [0.8.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.8.0...v0.8.1) - 2020-05-04
+
+### Fixed
+
+- Ignore failed jobs in Que adapter. ([#18](https://github.com/adamlogic/rails_autoscale_agent/pull/18))
+
+## [0.8.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.7.0...v0.8.0) - 2020-03-21
+
+### Added
+
+- Add support for Delayed Job ([#14](https://github.com/adamlogic/rails_autoscale_agent/pull/14))
+- Add support for Que ([#15](https://github.com/adamlogic/rails_autoscale_agent/pull/15))
+
+## [0.7.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.3...v0.7.0) - 2019-12-04
+
+### Added
+
+- Make worker adapters configurable. ([012d937](https://github.com/adamlogic/rails_autoscale_agent/commit/012d9379296763f5e42df95f05b066fe82ab0051))
+
+## [0.6.3](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.2...v0.6.3) - 2019-06-25
+
+### Fixed
+
+- Fix issues with logging.
+
+## [0.6.2](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.1...v0.6.2) - 2019-06-22
+
+### Fixed
+
+- Fix issues with logging.
+
+## [0.6.1](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.6.0...v0.6.1) - 2019-05-06
+
+### Fixed
+
+- Don't assume Sidekiq is present.
+
+## [0.6.0](https://github.com/adamlogic/rails_autoscale_agent/compare/v0.4.1...v0.6.0) - 2019-05-03
+
+### Added
+
+- Add support for autoscaling Sidekiq.

data/Gemfile

@@ -1,4 +1,16 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in rails_autoscale_agent.gemspec
 gemspec
+
+gem "rake", ">= 12.3.3"
+gem "rspec", ">= 3.0"
+gem "vcr", ">= 3.0"
+gem "webmock"
+gem "pry-byebug"
+gem "sidekiq", ">= 5.0"
+gem "delayed_job"
+gem "delayed_job_active_record"
+gem "que"
+gem "resque"
+gem "activesupport"
+gem "sqlite3", platforms: :ruby

data/README.md

@@ -2,11 +2,12 @@
 
 [![Build Status](https://travis-ci.org/adamlogic/rails_autoscale_agent.svg?branch=master)](https://travis-ci.org/adamlogic/rails_autoscale_agent)
 
-This gem works together with the [Rails Autoscale](https://railsautoscale.com) Heroku add-on to automatically scale your web dynos as needed. It gathers a minimal set of metrics for each request, and periodically posts this data asynchronously to the Rails Autoscale service.
+This gem works together with the [Rails Autoscale](https://railsautoscale.com) Heroku add-on to automatically scale your web and worker dynos as needed. It gathers a minimal set of metrics for each request (and job queue), and periodically posts this data asynchronously to the Rails Autoscale service.
 
 ## Requirements
 
-Tested with Rails versions 3.2 and higher and Ruby versions 1.9.3 and higher.
+- Rack-based app
+- Ruby 2.5 or newer
 
 ## Getting Started
 
@@ -22,15 +23,7 @@ The agent will only communicate with Rails Autoscale if a `RAILS_AUTOSCALE_URL`
 
 ## Non-Rails Rack apps
 
-You'll need to insert the `RailsAutoscaleAgent::Middleware` manually. Insert it before `Rack::Runtime` to ensure accuracy of request queue timings.
-
-## Changing the logger
-
-If you wish to use a different logger you can set it on the configuration object:
-
-```ruby
-RailsAutoscaleAgent::Config.instance.logger = MyLogger.new
-```
+You'll need to `require 'rails_autoscale_agent/middleware'` and insert the `RailsAutoscaleAgent::Middleware` manually. Insert it before `Rack::Runtime` to ensure accuracy of request queue timings.
 
 ## What data is collected?
 
@@ -43,29 +36,84 @@ The middleware agent runs in its own thread so your web requests are not impacte
 - PID
 - Collection of queue time measurements (time and milliseconds)
 
-Rails Autoscale processes and stores this information in order to power the autoscaling algorithm and dashboard visualizations.
+Rails Autoscale aggregates and stores this information to power the autoscaling algorithm and dashboard visualizations.
 
-## Troubleshooting
+## Configuration
+
+Most Rails Autoscale configurations are handled via the settings page on your Rails Autoscale dashboard, but there a few ways you can directly change the behavior of the agent via environment variables:
+
+- `RAILS_AUTOSCALE_DEBUG` - Enables debug logging. See more in the [logging](#logging) section below.
+- `RAILS_AUTOSCALE_WORKER_ADAPTER` - Overrides the available worker adapters. See more in the [worker adapters](#worker_adapters) section below.
+- `RAILS_AUTOSCALE_LONG_JOBS` - Enables reporting for active workers. See [Handling Long-Running Background Jobs](https://railsautoscale.com/docs/long-running-jobs/) in the Rails Autoscale docs for more.
+
+## Worker adapters
+
+Rails Autoscale supports autoscaling worker dynos. Out of the box, four job backends are supported: Sidekiq, Resque, Delayed Job, and Que. The agent will automatically enable the appropriate worker adapter based on what you have installed in your app.
+
+In some scenarios you might want to override this behavior. Let's say you have both Sidekiq and Resque installed 🤷‍♂️, but you only want Rails Autoscale to collect metrics for Sidekiq. Here's how you'd override that:
+
+```
+heroku config:add RAILS_AUTOSCALE_WORKER_ADAPTER=sidekiq
+```
+
+You can also disable collection of worker metrics altogether:
+
+```
+heroku config:add RAILS_AUTOSCALE_WORKER_ADAPTER=""
+```
+
+It's also possible to write a custom worker adapter. See [these docs](https://railsautoscale.com/docs/custom-worker-adapter/) for details.
 
-If your logger supports tagged logging (as the Rails logger does by default), all log output from this gem is prefixed with "[RailsAutoscale]".
+## Troubleshooting
 
-Once installed, you should see something like this in development:
+Once installed, you should see something like this in your development log:
 
 > [RailsAutoscale] Reporter not started: RAILS_AUTOSCALE_URL is not set
 
-In production, you should see something like this:
+In production, run `heroku logs -t | grep RailsAutoscale`, and you should see something like this:
 
 > [RailsAutoscale] Reporter starting, will report every 15 seconds
 
 If you don't see either of these, try running `bundle` again and restarting your Rails application.
 
+You can see more detailed (debug) logging by setting `RAILS_AUTOSCALE_DEBUG` on your Heroku app:
+
+```
+heroku config:add RAILS_AUTOSCALE_DEBUG=true
+```
+
+See more in the [logging](#logging) section below.
+
 Reach out to help@railsautoscale.com if you run into any other problems.
 
+## Logging
+
+The Rails logger is used by default.
+If you wish to use a different logger you can set it on the configuration object:
+
+```ruby
+# config/initializers/rails_autoscale_agent.rb
+RailsAutoscaleAgent::Config.instance.logger = MyLogger.new
+```
+
+Debug logs are silenced by default because Rails apps default to a DEBUG log level in production, and this gem has _very_ chatty debug logs. If you want to see the debug logs, set `RAILS_AUTOSCALE_DEBUG` on your Heroku app:
+
+```
+heroku config:add RAILS_AUTOSCALE_DEBUG=true
+```
+
+If you find the gem too chatty even without this, you can quiet it down further:
+
+```ruby
+# config/initializers/rails_autoscale_agent.rb
+RailsAutoscaleAgent::Config.instance.quiet = true
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, commit it, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/lib/rails_autoscale_agent/autoscale_api.rb

@@ -11,8 +11,8 @@ module RailsAutoscaleAgent
 
     SUCCESS = 'success'
 
-    def initialize(api_url_base)
-      @api_url_base = api_url_base
+    def initialize(config)
+      @config = config
     end
 
     def report_metrics!(report_params, timings_csv)
@@ -24,6 +24,10 @@ module RailsAutoscaleAgent
       post_json '/registrations', registration: registration_params
     end
 
+    def report_exception!(ex)
+      post_json '/exceptions', message: ex.inspect, backtrace: ex.backtrace.join("\n")
+    end
+
     private
 
     def post_json(path, data)
@@ -37,9 +41,14 @@ module RailsAutoscaleAgent
     end
 
     def post_raw(options)
-      uri = URI.parse("#{@api_url_base}#{options.fetch(:path)}")
+      uri = URI.parse("#{@config.api_base_url}#{options.fetch(:path)}")
       ssl = uri.scheme == 'https'
 
+      if @config.dev_mode
+        logger.debug "[DEV_MODE] Skipping request to #{uri}"
+        return SuccessResponse.new('{}')
+      end
+
       response = Net::HTTP.start(uri.host, uri.port, use_ssl: ssl) do |http|
         request = Net::HTTP::Post.new(uri.request_uri, options[:headers] || {})
         request.body = options.fetch(:body)
@@ -50,7 +59,7 @@ module RailsAutoscaleAgent
 
       case response.code.to_i
       when 200...300 then SuccessResponse.new(response.body)
-      else FailureResponse.new(response.message)
+      else FailureResponse.new([response.code, response.message].join(' - '))
       end
     end
 

data/lib/rails_autoscale_agent/config.rb

@@ -4,32 +4,53 @@ require 'singleton'
 
 module RailsAutoscaleAgent
   class Config
+    DEFAULT_WORKER_ADAPTERS = 'sidekiq,delayed_job,que,resque'
+
     include Singleton
 
     attr_accessor :report_interval, :logger, :api_base_url, :max_request_size,
-                  :dyno, :pid, :addon_name, :worker_adapters
+                  :dyno, :addon_name, :worker_adapters, :dev_mode, :debug, :quiet,
+                  :track_long_running_jobs,
+
+                  # legacy configs, no longer used
+                  :sidekiq_latency_for_active_jobs, :latency_for_active_jobs
 
     def initialize
+      @worker_adapters = prepare_worker_adapters
+
       # Allow the add-on name to be configured - needed for testing
       @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
       @api_base_url = ENV["#{@addon_name}_URL"]
-      @pid = Process.pid
+      @dev_mode = ENV['RAILS_AUTOSCALE_DEV'] == 'true'
+      @debug = dev_mode? || ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
+      @track_long_running_jobs = ENV['RAILS_AUTOSCALE_LONG_JOBS'] == 'true'
       @max_request_size = 100_000 # ignore request payloads over 100k since they skew the queue times
-      @report_interval = 60 # this default will be overwritten during Reporter#register!
+      @report_interval = 10 # this default will be overwritten during Reporter#register!
       @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
-      @dyno = ENV['DYNO']
-      @worker_adapters = [
-        WorkerAdapters::Sidekiq.new,
-      ]
+      @dyno = dev_mode? ? 'dev.1' : ENV['DYNO']
     end
 
     def to_s
-      "#{@dyno}##{@pid}"
+      "#{@dyno}##{Process.pid}"
     end
 
     def ignore_large_requests?
       @max_request_size
     end
 
+    alias_method :dev_mode?, :dev_mode
+    alias_method :debug?, :debug
+    alias_method :quiet?, :quiet
+
+    private
+
+    def prepare_worker_adapters
+      adapter_names = (ENV['RAILS_AUTOSCALE_WORKER_ADAPTER'] || DEFAULT_WORKER_ADAPTERS).split(',')
+      adapter_names.map do |adapter_name|
+        require "rails_autoscale_agent/worker_adapters/#{adapter_name}"
+        adapter_constant_name = adapter_name.capitalize.gsub(/(?:_)(.)/i) { $1.upcase }
+        WorkerAdapters.const_get(adapter_constant_name).instance
+      end
+    end
   end
 end