rails_autoscale_agent 0.6.3 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 152d6241b81f0efb42554a0a4ec1c842754cc4ad7cfc3dbf50a406695e59311c
-  data.tar.gz: 3deac83a1ab16e6e79c4f3d2fd1887ae8a71823516b254978fe479c7e00ea541
+  metadata.gz: 3924fc840320be8327c183e1cb54055d8516d2fefee1cffbfa7092da39b05985
+  data.tar.gz: abd81e6c4c2dc3a65be7c426dda8742ec92b7c29c47a71eeb2566731436ce4e6
 SHA512:
-  metadata.gz: 488c42d09b52e2cd5c9c7fee3d5c63b2fc3e7398cc1321d85e897478e9ab28407e17c2a0b12b5741178fcf618bb1b8aa6decb53b55823ab6e461d6f5cdf3095c
-  data.tar.gz: 61952a1aa625b25da6580a85300c7f3feaf6b5fe56b26eebbb9294e5ff3a9f53a9c6a4952bfab39a17089df8d67cd3c746786b3fa0d117c9ce8f0b9e27b12a4d
+  metadata.gz: 4867868c9e9e707d9e4afe46f20d19f600a87b26aabf5a96f4ad8ff91698d00b08f8a0566ef20fe8f3f1c036c9220b1034a92c70280a4cf29a6d5b5a728b4105
+  data.tar.gz: e2fbbc4afa90c0c636dddb3172d1164108e984f4c550739df552cd185a8a51d7255ff354debc658ecfa893fb4f87f53e6c298bfe4dddf2aa35615f7636e64420

data/.ruby-version

@@ -1 +1 @@
-2.5.0
+2.6.5

data/.vscode/tasks.json

@@ -0,0 +1,84 @@
+{
+  // See https://go.microsoft.com/fwlink/?LinkId=733558
+  // for the documentation about the tasks.json format
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "test: all",
+      "group": "test",
+      "command": "rspec",
+      "runOptions": {
+        "reevaluateOnRerun": false
+      },
+      "problemMatcher": {
+        "owner": "ruby",
+        "fileLocation": ["relative", "${workspaceRoot}"],
+        "severity": "error",
+        "pattern": [
+          {
+            "regexp": "^\\s{7}(.*)$",
+            "message": 1
+          },
+          {
+            "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
+            "file": 1,
+            "line": 2,
+            "code": 3
+          }
+        ]
+      }
+    },
+    {
+      "label": "test: file",
+      "group": "test",
+      "command": "rspec",
+      "args": ["${relativeFile}"],
+      "runOptions": {
+        "reevaluateOnRerun": false
+      },
+      "problemMatcher": {
+        "owner": "ruby",
+        "fileLocation": ["relative", "${workspaceRoot}"],
+        "severity": "error",
+        "pattern": [
+          {
+            "regexp": "^\\s{7}(.*)$",
+            "message": 1
+          },
+          {
+            "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
+            "file": 1,
+            "line": 2,
+            "code": 3
+          }
+        ]
+      }
+    },
+    {
+      "label": "test: line",
+      "group": "test",
+      "command": "rspec",
+      "args": ["${relativeFile}:${lineNumber}"],
+      "runOptions": {
+        "reevaluateOnRerun": false
+      },
+      "problemMatcher": {
+        "owner": "ruby",
+        "fileLocation": ["relative", "${workspaceRoot}"],
+        "severity": "error",
+        "pattern": [
+          {
+            "regexp": "^\\s{7}(.*)$",
+            "message": 1
+          },
+          {
+            "regexp": "^\\s+#\\s(\\./.*):(\\d+):(in.*)$",
+            "file": 1,
+            "line": 2,
+            "code": 3
+          }
+        ]
+      }
+    }
+  ]
+}

data/README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/adamlogic/rails_autoscale_agent.svg?branch=master)](https://travis-ci.org/adamlogic/rails_autoscale_agent)
 
-This gem works together with the [Rails Autoscale](https://railsautoscale.com) Heroku add-on to automatically scale your web dynos as needed. It gathers a minimal set of metrics for each request, and periodically posts this data asynchronously to the Rails Autoscale service.
+This gem works together with the [Rails Autoscale](https://railsautoscale.com) Heroku add-on to automatically scale your web and worker dynos as needed. It gathers a minimal set of metrics for each request (and job queue), and periodically posts this data asynchronously to the Rails Autoscale service.
 
 ## Requirements
 
@@ -26,9 +26,11 @@ You'll need to insert the `RailsAutoscaleAgent::Middleware` manually. Insert it
 
 ## Changing the logger
 
+The Rails logger is used by default.
 If you wish to use a different logger you can set it on the configuration object:
 
 ```ruby
+# config/initializers/rails_autoscale_agent.rb
 RailsAutoscaleAgent::Config.instance.logger = MyLogger.new
 ```
 
@@ -43,22 +45,29 @@ The middleware agent runs in its own thread so your web requests are not impacte
 - PID
 - Collection of queue time measurements (time and milliseconds)
 
-Rails Autoscale processes and stores this information in order to power the autoscaling algorithm and dashboard visualizations.
+Rails Autoscale aggregates and stores this information to power the autoscaling algorithm and dashboard visualizations.
 
 ## Troubleshooting
 
-If your logger supports tagged logging (as the Rails logger does by default), all log output from this gem is prefixed with "[RailsAutoscale]".
-
-Once installed, you should see something like this in development:
+Once installed, you should see something like this in your development log:
 
 > [RailsAutoscale] Reporter not started: RAILS_AUTOSCALE_URL is not set
 
-In production, you should see something like this:
+In production, run `heroku logs -t | grep RailsAutoscale`, and you should see something like this:
 
 > [RailsAutoscale] Reporter starting, will report every 15 seconds
 
 If you don't see either of these, try running `bundle` again and restarting your Rails application.
 
+You can see more detailed (debug) logging by setting the `RAILS_AUTOSCALE_DEBUG` env var on your Heroku app:
+
+```
+heroku config:add RAILS_AUTOSCALE_DEBUG=true
+```
+
+Debug logs are silenced by default because Rails apps default to a DEBUG log level in production,
+and these can get very noisy with this gem.
+
 Reach out to help@railsautoscale.com if you run into any other problems.
 
 ## Development

data/lib/rails_autoscale_agent/config.rb

@@ -7,9 +7,18 @@ module RailsAutoscaleAgent
     include Singleton
 
     attr_accessor :report_interval, :logger, :api_base_url, :max_request_size,
-                  :dyno, :pid, :addon_name
+                  :dyno, :pid, :addon_name, :worker_adapters
 
     def initialize
+      require 'rails_autoscale_agent/worker_adapters/sidekiq'
+      require 'rails_autoscale_agent/worker_adapters/delayed_job'
+      require 'rails_autoscale_agent/worker_adapters/que'
+      @worker_adapters = [
+        WorkerAdapters::Sidekiq.instance,
+        WorkerAdapters::DelayedJob.instance,
+        WorkerAdapters::Que.instance,
+      ]
+
       # Allow the add-on name to be configured - needed for testing
       @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
       @api_base_url = ENV["#{@addon_name}_URL"]

data/lib/rails_autoscale_agent/logger.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/module/delegation'
 require 'rails_autoscale_agent/config'
+require 'logger'
 
 module RailsAutoscaleAgent
   module Logger
@@ -11,23 +11,30 @@ module RailsAutoscaleAgent
   end
 
   class LoggerProxy < Struct.new(:logger)
-    def tagged(*tags, &block)
-      if logger.respond_to?(:tagged)
-        logger.tagged *tags, &block
-      else
-        # NOTE: Quack like ActiveSupport::TaggedLogging, but don't reimplement
-        yield self
+    TAG = '[RailsAutoscale]'
+
+    %w[info warn error].each do |name|
+      define_method name do |msg|
+        logger.send name, tag(msg)
       end
     end
 
-    def debug(*args)
-      # Rails logger defaults to DEBUG level in production, but I don't want
-      # to be chatty by default.
-      logger.debug(*args) if ENV['RAILS_AUTOSCALE_LOG_LEVEL'] == 'DEBUG'
+    def debug(msg)
+      # Silence debug logs by default to avoiding being overly chatty (Rails logger defaults
+      # to DEBUG level in production).
+      # This uses a separate logger so that RAILS_AUTOSCALE_DEBUG
+      # shows debug logs regardless of Rails log level.
+      debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
+    end
+
+    private
+
+    def debug_logger
+      @debug_loggers ||= ::Logger.new(STDOUT)
     end
 
-    def method_missing(name, *args, &block)
-      logger.send name, *args, &block
+    def tag(msg)
+      "#{TAG} #{msg}"
     end
   end
 end

data/lib/rails_autoscale_agent/middleware.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'rails_autoscale_agent/logger'
 require 'rails_autoscale_agent/store'
 require 'rails_autoscale_agent/reporter'
 require 'rails_autoscale_agent/config'
@@ -8,25 +7,21 @@ require 'rails_autoscale_agent/request'
 
 module RailsAutoscaleAgent
   class Middleware
-    include Logger
-
     def initialize(app)
       @app = app
     end
 
     def call(env)
-      logger.tagged 'RailsAutoscale' do
-        config = Config.instance
-        request = Request.new(env, config)
+      config = Config.instance
+      request = Request.new(env, config)
 
-        store = Store.instance
-        Reporter.start(config, store)
+      store = Store.instance
+      Reporter.start(config, store)
 
-        if !request.ignore? && queue_time = request.queue_time
-          # NOTE: Expose queue time to the app
-          env['queue_time'] = queue_time
-          store.push queue_time
-        end
+      if !request.ignore? && queue_time = request.queue_time
+        # NOTE: Expose queue time to the app
+        env['queue_time'] = queue_time
+        store.push queue_time
       end
 
       @app.call(env)

data/lib/rails_autoscale_agent/railtie.rb

@@ -1,10 +1,14 @@
 # frozen_string_literal: true
 
 require 'rails_autoscale_agent/middleware'
+require 'rails_autoscale_agent/logger'
 
 module RailsAutoscaleAgent
   class Railtie < Rails::Railtie
+    include Logger
+
     initializer "rails_autoscale_agent.middleware" do |app|
+      logger.info "Preparing middleware"
       app.middleware.insert_before Rack::Runtime, Middleware
     end
   end

data/lib/rails_autoscale_agent/reporter.rb

@@ -5,7 +5,6 @@ require 'rails_autoscale_agent/logger'
 require 'rails_autoscale_agent/autoscale_api'
 require 'rails_autoscale_agent/time_rounder'
 require 'rails_autoscale_agent/registration'
-require 'rails_autoscale_agent/worker_adapters/sidekiq'
 
 # Reporter wakes up every minute to send metrics to the RailsAutoscale API
 
@@ -14,17 +13,13 @@ module RailsAutoscaleAgent
     include Singleton
     include Logger
 
-    WORKER_ADAPTERS = [
-      WorkerAdapters::Sidekiq.new,
-    ]
-
     def self.start(config, store)
       instance.start!(config, store) unless instance.started?
     end
 
     def start!(config, store)
       @started = true
-      @worker_adapters = WORKER_ADAPTERS.select(&:enabled?)
+      @worker_adapters = config.worker_adapters.select(&:enabled?)
 
       if !config.api_base_url
         logger.info "Reporter not started: #{config.addon_name}_URL is not set"
@@ -32,23 +27,21 @@ module RailsAutoscaleAgent
       end
 
       Thread.new do
-        logger.tagged 'RailsAutoscale' do
-          register!(config)
-
-          loop do
-            # Stagger reporting to spread out reports from many processes
-            multiplier = 1 - (rand / 4) # between 0.75 and 1.0
-            sleep config.report_interval * multiplier
-
-            begin
-              @worker_adapters.map { |a| a.collect!(store) }
-              report!(config, store)
-            rescue => ex
-              # Exceptions in threads other than the main thread will fail silently
-              # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
-              logger.error "Reporter error: #{ex.inspect}"
-              logger.error ex.backtrace.join("\n")
-            end
+        loop do
+          register!(config) unless @registered
+
+          # Stagger reporting to spread out reports from many processes
+          multiplier = 1 - (rand / 4) # between 0.75 and 1.0
+          sleep config.report_interval * multiplier
+
+          begin
+            @worker_adapters.map { |a| a.collect!(store) }
+            report!(config, store)
+          rescue => ex
+            # Exceptions in threads other than the main thread will fail silently
+            # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
+            logger.error "Reporter error: #{ex.inspect}"
+            logger.error ex.backtrace.join("\n")
           end
         end
       end
@@ -84,13 +77,14 @@ module RailsAutoscaleAgent
 
       case result
       when AutoscaleApi::SuccessResponse
+        @registered = true
         config.report_interval = result.data['report_interval'] if result.data['report_interval']
         config.max_request_size = result.data['max_request_size'] if result.data['max_request_size']
-        logger.info "Reporter starting, will report every #{config.report_interval} seconds or so"
+        worker_adapters_msg = @worker_adapters.map { |a| a.class.name }.join(', ')
+        logger.info "Reporter starting, will report every #{config.report_interval} seconds or so. Worker adapters: [#{worker_adapters_msg}]"
       when AutoscaleApi::FailureResponse
         logger.error "Reporter failed to register: #{result.failure_message}"
       end
     end
-
   end
 end

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.6.3"
+  VERSION = "0.8.3"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+
+module WorkerAdapters
+  class DelayedJob
+    include RailsAutoscaleAgent::Logger
+    include Singleton
+
+    class << self
+      attr_accessor :queues
+    end
+
+    def initialize
+      # Track the known queues so we can continue reporting on queues that don't
+      # currently have enqueued jobs.
+      self.class.queues = Set.new
+
+      install if enabled?
+    end
+
+    def enabled?
+      defined? ::Delayed
+    end
+
+    def collect!(store)
+      log_msg = String.new('DelayedJob latency ')
+      t = Time.now
+
+      # Ignore failed jobs (they skew latency measurement due to the original run_at)
+      sql = 'SELECT queue, min(run_at) FROM delayed_jobs WHERE attempts = 0 GROUP BY queue'
+      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+      queues = self.class.queues | run_at_by_queue.keys
+
+      queues.each do |queue|
+        next if queue.nil? || queue.empty?
+        run_at = run_at_by_queue[queue]
+        run_at = Time.parse(run_at) if run_at.is_a?(String)
+        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+        store.push latency_ms, t, queue
+        log_msg << "#{queue}=#{latency_ms} "
+      end
+
+      logger.debug log_msg
+    end
+
+    private
+
+    def install
+      plugin = Class.new(Delayed::Plugin) do
+        require 'delayed_job'
+
+        callbacks do |lifecycle|
+          lifecycle.before(:enqueue) do |job, &block|
+            queue = job.queue || 'default'
+            WorkerAdapters::DelayedJob.queues.add queue
+          end
+        end
+      end
+
+      Delayed::Worker.plugins << plugin
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+require 'time'
+
+module WorkerAdapters
+  class Que
+    include RailsAutoscaleAgent::Logger
+    include Singleton
+
+    DEFAULT_QUEUES = ['default']
+
+    class << self
+      attr_accessor :queues
+    end
+
+    def initialize
+      self.class.queues = DEFAULT_QUEUES
+    end
+
+    def enabled?
+      defined? ::Que
+    end
+
+    def collect!(store)
+      log_msg = String.new('Que latency ')
+      t = Time.now
+
+      # Ignore failed jobs (they skew latency measurement due to the original run_at)
+      sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
+      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+      self.class.queues |= run_at_by_queue.keys
+
+      self.class.queues.each do |queue|
+        next if queue.nil? || queue.empty?
+        run_at = run_at_by_queue[queue]
+        run_at = Time.parse(run_at) if run_at.is_a?(String)
+        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+        store.push latency_ms, t, queue
+        log_msg << "#{queue}=#{latency_ms} "
+      end
+
+      logger.debug log_msg
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -5,6 +5,7 @@ require 'rails_autoscale_agent/logger'
 module WorkerAdapters
   class Sidekiq
     include RailsAutoscaleAgent::Logger
+    include Singleton
 
     def enabled?
       require 'sidekiq/api'
@@ -13,7 +14,6 @@ module WorkerAdapters
       false
     end
 
-    # TODO: specs
     def collect!(store)
       log_msg = String.new('Sidekiq latency ')
 

data/rails_autoscale_agent.gemspec

@@ -16,14 +16,15 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport", ">= 3.2"
-
   spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "vcr", "~> 3.0"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "sidekiq", "~> 5.0"
+  spec.add_development_dependency "delayed_job"
+  spec.add_development_dependency "que"
+  spec.add_development_dependency "activesupport"
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.8.3
 platform: ruby
 authors:
 - Adam McCrea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-25 00:00:00.000000000 Z
+date: 2020-05-26 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: que
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - adam@adamlogic.com
@@ -147,6 +175,7 @@ files:
 - ".rspec"
 - ".ruby-version"
 - ".travis.yml"
+- ".vscode/tasks.json"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -167,6 +196,8 @@ files:
 - lib/rails_autoscale_agent/store.rb
 - lib/rails_autoscale_agent/time_rounder.rb
 - lib/rails_autoscale_agent/version.rb
+- lib/rails_autoscale_agent/worker_adapters/delayed_job.rb
+- lib/rails_autoscale_agent/worker_adapters/que.rb
 - lib/rails_autoscale_agent/worker_adapters/sidekiq.rb
 - log/.gitkeep
 - rails_autoscale_agent.gemspec
@@ -189,8 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: This gem works with the Rails Autoscale Heroku add-on to automatically scale