rails_autoscale_agent 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 355209571af7f1f6c5fbe38ed112a2936903afcdd9cff56b9c68531be45c8d5e
-  data.tar.gz: 9855ff92fcc103546a4fe3115942b4e94e0c5e3e626d6b14b26c1eeab0953bc8
+  metadata.gz: ecfcd2368450e449cfcb8b644621a19cd5f8d14a5d25bd2f6f85383c2e372b83
+  data.tar.gz: 61008635232925e88adaea78ecd33514e282dc678e5b6a8c46365bc5636402da
 SHA512:
-  metadata.gz: 1e76f3c52649b97c5919ce06170414e0d4f2642bf133f781ee3e54e842f77e0264c37f625e1f4c1c681ce79aec162bb7749d133a663e7d580cfc5385813f591b
-  data.tar.gz: c312bfb9654d06c25be2bda4da119e74e547fc24f0b71cd412fdeef63665154b37bf013beaae9ad1c4fb169469c0bff4cc0a601942bbe845bb739708ecc0d426
+  metadata.gz: 3ddbc592abd1d260081e764f5628a33d6c984d55229cde0da911fa4d56c49d45dc2cb8e89486882851bfc6041bca7797d6d3fea709b699edeb7c687bb4aa0adc
+  data.tar.gz: 47f1a329b95e49566725ba73e0266645a9966d657c5e64c7afec536a02fde4fcc6c3f258ccf373db909f47698ab47bac7bd1471b373485e31ae716436b267542

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/adamlogic/rails_autoscale_agent.svg?branch=master)](https://travis-ci.org/adamlogic/rails_autoscale_agent)
 
-This gem works together with the [Rails Autoscale](https://railsautoscale.com) Heroku add-on to automatically scale your web dynos as needed. It gathers a minimal set of metrics for each request, and periodically posts this data asynchronously to the Rails Autoscale service.
+This gem works together with the [Rails Autoscale](https://railsautoscale.com) Heroku add-on to automatically scale your web and worker dynos as needed. It gathers a minimal set of metrics for each request (and job queue), and periodically posts this data asynchronously to the Rails Autoscale service.
 
 ## Requirements
 
@@ -26,9 +26,11 @@ You'll need to insert the `RailsAutoscaleAgent::Middleware` manually. Insert it
 
 ## Changing the logger
 
+The Rails logger is used by default.
 If you wish to use a different logger you can set it on the configuration object:
 
 ```ruby
+# config/initializers/rails_autoscale_agent.rb
 RailsAutoscaleAgent::Config.instance.logger = MyLogger.new
 ```
 
@@ -43,22 +45,29 @@ The middleware agent runs in its own thread so your web requests are not impacte
 - PID
 - Collection of queue time measurements (time and milliseconds)
 
-Rails Autoscale processes and stores this information in order to power the autoscaling algorithm and dashboard visualizations.
+Rails Autoscale aggregates and stores this information to power the autoscaling algorithm and dashboard visualizations.
 
 ## Troubleshooting
 
-If your logger supports tagged logging (as the Rails logger does by default), all log output from this gem is prefixed with "[RailsAutoscale]".
-
-Once installed, you should see something like this in development:
+Once installed, you should see something like this in your development log:
 
 > [RailsAutoscale] Reporter not started: RAILS_AUTOSCALE_URL is not set
 
-In production, you should see something like this:
+In production, run `heroku logs -t | grep RailsAutoscale`, and you should see something like this:
 
 > [RailsAutoscale] Reporter starting, will report every 15 seconds
 
 If you don't see either of these, try running `bundle` again and restarting your Rails application.
 
+You can see more detailed (debug) logging by setting the `RAILS_AUTOSCALE_DEBUG` env var on your Heroku app:
+
+```
+heroku config:add RAILS_AUTOSCALE_DEBUG=true
+```
+
+Debug logs are silenced by default because Rails apps default to a DEBUG log level in production,
+and these can get very noisy with this gem.
+
 Reach out to help@railsautoscale.com if you run into any other problems.
 
 ## Development

data/lib/rails_autoscale_agent/config.rb

@@ -10,6 +10,15 @@ module RailsAutoscaleAgent
                   :dyno, :pid, :addon_name, :worker_adapters
 
     def initialize
+      require 'rails_autoscale_agent/worker_adapters/sidekiq'
+      require 'rails_autoscale_agent/worker_adapters/delayed_job'
+      require 'rails_autoscale_agent/worker_adapters/que'
+      @worker_adapters = [
+        WorkerAdapters::Sidekiq.instance,
+        WorkerAdapters::DelayedJob.instance,
+        WorkerAdapters::Que.instance,
+      ]
+
       # Allow the add-on name to be configured - needed for testing
       @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
       @api_base_url = ENV["#{@addon_name}_URL"]
@@ -18,9 +27,6 @@ module RailsAutoscaleAgent
       @report_interval = 60 # this default will be overwritten during Reporter#register!
       @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
       @dyno = ENV['DYNO']
-      @worker_adapters = [
-        WorkerAdapters::Sidekiq.new,
-      ]
     end
 
     def to_s

data/lib/rails_autoscale_agent/logger.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/module/delegation'
 require 'rails_autoscale_agent/config'
+require 'logger'
 
 module RailsAutoscaleAgent
   module Logger
@@ -11,23 +11,30 @@ module RailsAutoscaleAgent
   end
 
   class LoggerProxy < Struct.new(:logger)
-    def tagged(*tags, &block)
-      if logger.respond_to?(:tagged)
-        logger.tagged *tags, &block
-      else
-        # NOTE: Quack like ActiveSupport::TaggedLogging, but don't reimplement
-        yield self
+    TAG = '[RailsAutoscale]'
+
+    %w[info warn error].each do |name|
+      define_method name do |msg|
+        logger.send name, tag(msg)
       end
     end
 
-    def debug(*args)
-      # Rails logger defaults to DEBUG level in production, but I don't want
-      # to be chatty by default.
-      logger.debug(*args) if ENV['RAILS_AUTOSCALE_LOG_LEVEL'] == 'DEBUG'
+    def debug(msg)
+      # Silence debug logs by default to avoiding being overly chatty (Rails logger defaults
+      # to DEBUG level in production).
+      # This uses a separate logger so that RAILS_AUTOSCALE_DEBUG
+      # shows debug logs regardless of Rails log level.
+      debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
+    end
+
+    private
+
+    def debug_logger
+      @debug_loggers ||= ::Logger.new(STDOUT)
     end
 
-    def method_missing(name, *args, &block)
-      logger.send name, *args, &block
+    def tag(msg)
+      "#{TAG} #{msg}"
     end
   end
 end

data/lib/rails_autoscale_agent/middleware.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'rails_autoscale_agent/logger'
 require 'rails_autoscale_agent/store'
 require 'rails_autoscale_agent/reporter'
 require 'rails_autoscale_agent/config'
@@ -8,25 +7,21 @@ require 'rails_autoscale_agent/request'
 
 module RailsAutoscaleAgent
   class Middleware
-    include Logger
-
     def initialize(app)
       @app = app
     end
 
     def call(env)
-      logger.tagged 'RailsAutoscale' do
-        config = Config.instance
-        request = Request.new(env, config)
+      config = Config.instance
+      request = Request.new(env, config)
 
-        store = Store.instance
-        Reporter.start(config, store)
+      store = Store.instance
+      Reporter.start(config, store)
 
-        if !request.ignore? && queue_time = request.queue_time
-          # NOTE: Expose queue time to the app
-          env['queue_time'] = queue_time
-          store.push queue_time
-        end
+      if !request.ignore? && queue_time = request.queue_time
+        # NOTE: Expose queue time to the app
+        env['queue_time'] = queue_time
+        store.push queue_time
       end
 
       @app.call(env)

data/lib/rails_autoscale_agent/railtie.rb

@@ -1,10 +1,14 @@
 # frozen_string_literal: true
 
 require 'rails_autoscale_agent/middleware'
+require 'rails_autoscale_agent/logger'
 
 module RailsAutoscaleAgent
   class Railtie < Rails::Railtie
+    include Logger
+
     initializer "rails_autoscale_agent.middleware" do |app|
+      logger.info "Preparing middleware"
       app.middleware.insert_before Rack::Runtime, Middleware
     end
   end

data/lib/rails_autoscale_agent/reporter.rb

@@ -5,7 +5,6 @@ require 'rails_autoscale_agent/logger'
 require 'rails_autoscale_agent/autoscale_api'
 require 'rails_autoscale_agent/time_rounder'
 require 'rails_autoscale_agent/registration'
-require 'rails_autoscale_agent/worker_adapters/sidekiq'
 
 # Reporter wakes up every minute to send metrics to the RailsAutoscale API
 
@@ -28,23 +27,21 @@ module RailsAutoscaleAgent
       end
 
       Thread.new do
-        logger.tagged 'RailsAutoscale' do
-          register!(config)
-
-          loop do
-            # Stagger reporting to spread out reports from many processes
-            multiplier = 1 - (rand / 4) # between 0.75 and 1.0
-            sleep config.report_interval * multiplier
-
-            begin
-              @worker_adapters.map { |a| a.collect!(store) }
-              report!(config, store)
-            rescue => ex
-              # Exceptions in threads other than the main thread will fail silently
-              # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
-              logger.error "Reporter error: #{ex.inspect}"
-              logger.error ex.backtrace.join("\n")
-            end
+        loop do
+          register!(config) unless @registered
+
+          # Stagger reporting to spread out reports from many processes
+          multiplier = 1 - (rand / 4) # between 0.75 and 1.0
+          sleep config.report_interval * multiplier
+
+          begin
+            @worker_adapters.map { |a| a.collect!(store) }
+            report!(config, store)
+          rescue => ex
+            # Exceptions in threads other than the main thread will fail silently
+            # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
+            logger.error "Reporter error: #{ex.inspect}"
+            logger.error ex.backtrace.join("\n")
           end
         end
       end
@@ -80,6 +77,7 @@ module RailsAutoscaleAgent
 
       case result
       when AutoscaleApi::SuccessResponse
+        @registered = true
         config.report_interval = result.data['report_interval'] if result.data['report_interval']
         config.max_request_size = result.data['max_request_size'] if result.data['max_request_size']
         worker_adapters_msg = @worker_adapters.map { |a| a.class.name }.join(', ')
@@ -88,6 +86,5 @@ module RailsAutoscaleAgent
         logger.error "Reporter failed to register: #{result.failure_message}"
       end
     end
-
   end
 end

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+
+module WorkerAdapters
+  class DelayedJob
+    include RailsAutoscaleAgent::Logger
+    include Singleton
+
+    class << self
+      attr_accessor :queues
+    end
+
+    def initialize
+      # Track the known queues so we can continue reporting on queues that don't
+      # currently have enqueued jobs.
+      self.class.queues = Set.new
+
+      install if enabled?
+    end
+
+    def enabled?
+      defined? ::Delayed
+    end
+
+    def collect!(store)
+      log_msg = String.new('DelayedJob latency ')
+      t = Time.now
+
+      sql = 'SELECT queue, min(run_at) FROM delayed_jobs GROUP BY queue'
+      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+      queues = self.class.queues | run_at_by_queue.keys
+
+      queues.each do |queue|
+        run_at = run_at_by_queue[queue]
+        run_at = Time.parse(run_at) if run_at.is_a?(String)
+        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+        store.push latency_ms, t, queue
+        log_msg << "#{queue}=#{latency_ms} "
+      end
+
+      logger.debug log_msg
+    end
+
+    private
+
+    def install
+      plugin = Class.new(Delayed::Plugin) do
+        require 'delayed_job'
+
+        callbacks do |lifecycle|
+          lifecycle.before(:enqueue) do |job, &block|
+            queue = job.queue || 'default'
+            WorkerAdapters::DelayedJob.queues.add queue
+          end
+        end
+      end
+
+      Delayed::Worker.plugins << plugin
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+require 'time'
+
+module WorkerAdapters
+  class Que
+    include RailsAutoscaleAgent::Logger
+    include Singleton
+
+    DEFAULT_QUEUES = ['default']
+
+    class << self
+      attr_accessor :queues
+    end
+
+    def initialize
+      self.class.queues = DEFAULT_QUEUES
+    end
+
+    def enabled?
+      defined? ::Que
+    end
+
+    def collect!(store)
+      log_msg = String.new('Que latency ')
+      t = Time.now
+
+      sql = 'SELECT queue, min(run_at) FROM que_jobs GROUP BY queue'
+      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+      self.class.queues |= run_at_by_queue.keys
+
+      self.class.queues.each do |queue|
+        run_at = run_at_by_queue[queue]
+        run_at = Time.parse(run_at) if run_at.is_a?(String)
+        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+        store.push latency_ms, t, queue
+        log_msg << "#{queue}=#{latency_ms} "
+      end
+
+      logger.debug log_msg
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -5,6 +5,7 @@ require 'rails_autoscale_agent/logger'
 module WorkerAdapters
   class Sidekiq
     include RailsAutoscaleAgent::Logger
+    include Singleton
 
     def enabled?
       require 'sidekiq/api'
@@ -13,7 +14,6 @@ module WorkerAdapters
       false
     end
 
-    # TODO: specs
     def collect!(store)
       log_msg = String.new('Sidekiq latency ')
 

data/rails_autoscale_agent.gemspec

@@ -16,14 +16,15 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport", ">= 3.2"
-
   spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "vcr", "~> 3.0"
   spec.add_development_dependency "webmock"
-  spec.add_development_dependency "pry"
+  # pry 0.13.0 seems to be incompatible with pry-byebug 3.8.0
+  spec.add_development_dependency "pry", "0.12.2"
   spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "sidekiq", "~> 5.0"
+  spec.add_development_dependency "delayed_job"
+  spec.add_development_dependency "que"
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Adam McCrea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-04 00:00:00.000000000 Z
+date: 2020-03-21 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -98,16 +84,16 @@ dependencies:
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.2
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: que
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - adam@adamlogic.com
@@ -167,6 +181,8 @@ files:
 - lib/rails_autoscale_agent/store.rb
 - lib/rails_autoscale_agent/time_rounder.rb
 - lib/rails_autoscale_agent/version.rb
+- lib/rails_autoscale_agent/worker_adapters/delayed_job.rb
+- lib/rails_autoscale_agent/worker_adapters/que.rb
 - lib/rails_autoscale_agent/worker_adapters/sidekiq.rb
 - log/.gitkeep
 - rails_autoscale_agent.gemspec