rails_autoscale_agent 0.8.1 → 0.9.0.beta.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 131a1ca49bb43913967f4ba4b919d0e1072e7209c79a8c120fb497140535d5c4
-  data.tar.gz: a0c482dc7d1f4cc66375d2e37bdd4f5732cdf2b05095a37b93ef8a5332afff96
+  metadata.gz: 70253908c027308ec4d19c9b41eeed033759e6669eace3301925a65a64748447
+  data.tar.gz: 199e75a27f4708920b47964cbc75e2b705ddce88508c35fd614512bb9bfcfd5e
 SHA512:
-  metadata.gz: 3af632a6c5eb3022dac82655546b483ad183c0698ce8dc04e826dbeb5fae34d2c52deb7dc8b1a504c7c206f8a998953cd80fe5592c39ba5760aa4dd72948d9c6
-  data.tar.gz: aa8a4cea00a63a1d2e3581084cfbecdfe67ad182774a56503a43091427927da28baf54358ad12e5c218f07ff1dc237033cb2231fc3ca9da75d765a69f39e46a2
+  metadata.gz: af5043f8402b40c15b5efe0be205e0e4c6107b8d05ea7126ccfa445c28679567c7e0f15e2a01eb2b7e107c95d3802543c48f428c31f70be823953173cd903ae1
+  data.tar.gz: 1fcfa15d0c76a8f1abf38653890d6b2157fd2e4edfc2ac072eb198897ab09029ea3f1c83ee11620e13a1a07a379695d7913c5bed89ce51e1907451367e61b1df

data/README.md

@@ -74,7 +74,7 @@ Reach out to help@railsautoscale.com if you run into any other problems.
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, commit it, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/lib/rails_autoscale_agent/autoscale_api.rb

@@ -11,8 +11,8 @@ module RailsAutoscaleAgent
 
     SUCCESS = 'success'
 
-    def initialize(api_url_base)
-      @api_url_base = api_url_base
+    def initialize(config)
+      @config = config
     end
 
     def report_metrics!(report_params, timings_csv)
@@ -24,6 +24,10 @@ module RailsAutoscaleAgent
       post_json '/registrations', registration: registration_params
     end
 
+    def report_exception!(ex)
+      post_json '/exceptions', message: ex.inspect, backtrace: ex.backtrace.join("\n")
+    end
+
     private
 
     def post_json(path, data)
@@ -37,9 +41,14 @@ module RailsAutoscaleAgent
     end
 
     def post_raw(options)
-      uri = URI.parse("#{@api_url_base}#{options.fetch(:path)}")
+      uri = URI.parse("#{@config.api_base_url}#{options.fetch(:path)}")
       ssl = uri.scheme == 'https'
 
+      if @config.dev_mode
+        logger.debug "[DEV_MODE] Skipping request to #{uri}"
+        return SuccessResponse.new('{}')
+      end
+
       response = Net::HTTP.start(uri.host, uri.port, use_ssl: ssl) do |http|
         request = Net::HTTP::Post.new(uri.request_uri, options[:headers] || {})
         request.body = options.fetch(:body)

data/lib/rails_autoscale_agent/config.rb

@@ -7,26 +7,29 @@ module RailsAutoscaleAgent
     include Singleton
 
     attr_accessor :report_interval, :logger, :api_base_url, :max_request_size,
-                  :dyno, :pid, :addon_name, :worker_adapters
+                  :dyno, :pid, :addon_name, :worker_adapters, :dev_mode
 
     def initialize
       require 'rails_autoscale_agent/worker_adapters/sidekiq'
       require 'rails_autoscale_agent/worker_adapters/delayed_job'
       require 'rails_autoscale_agent/worker_adapters/que'
+      require 'rails_autoscale_agent/worker_adapters/resque'
       @worker_adapters = [
         WorkerAdapters::Sidekiq.instance,
         WorkerAdapters::DelayedJob.instance,
         WorkerAdapters::Que.instance,
+        WorkerAdapters::Resque.instance,
       ]
 
       # Allow the add-on name to be configured - needed for testing
       @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
       @api_base_url = ENV["#{@addon_name}_URL"]
+      @dev_mode = ENV['RAILS_AUTOSCALE_DEV'] == 'true'
       @pid = Process.pid
       @max_request_size = 100_000 # ignore request payloads over 100k since they skew the queue times
-      @report_interval = 60 # this default will be overwritten during Reporter#register!
+      @report_interval = 10 # this default will be overwritten during Reporter#register!
       @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
-      @dyno = ENV['DYNO']
+      @dyno = @dev_mode ? 'dev.1' : ENV['DYNO']
     end
 
     def to_s
@@ -37,5 +40,6 @@ module RailsAutoscaleAgent
       @max_request_size
     end
 
+    alias_method :dev_mode?, :dev_mode
   end
 end

data/lib/rails_autoscale_agent/logger.rb

@@ -24,7 +24,7 @@ module RailsAutoscaleAgent
       # to DEBUG level in production).
       # This uses a separate logger so that RAILS_AUTOSCALE_DEBUG
       # shows debug logs regardless of Rails log level.
-      debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
+      debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true' || Config.instance.dev_mode?
     end
 
     private

data/lib/rails_autoscale_agent/measurement.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  class Measurement < Struct.new(:time, :value, :queue_name)
-    def initialize(time, value, queue_name = nil)
-      super time.utc, value.to_i, queue_name
+  class Measurement < Struct.new(:time, :value, :queue_name, :metric)
+    # No queue_name is assumed to be a web request measurement
+    # Metrics: qt = queue time (default), qd = queue depth (needed for Resque support)
+    def initialize(time, value, queue_name = nil, metric = nil)
+      super time.utc, value.to_i, queue_name, metric
     end
   end
 end

data/lib/rails_autoscale_agent/registration.rb

@@ -3,7 +3,7 @@
 require 'rails_autoscale_agent/version'
 
 module RailsAutoscaleAgent
-  class Registration < Struct.new(:config)
+  class Registration < Struct.new(:config, :worker_adapters)
 
     def to_params
       {
@@ -12,6 +12,8 @@ module RailsAutoscaleAgent
         ruby_version: RUBY_VERSION,
         rails_version: defined?(Rails) && Rails.version,
         gem_version: VERSION,
+        # example: { worker_adapters: 'Sidekiq,Que' }
+        worker_adapters: worker_adapters.map { |o| o.class.name.split('::').last }.join(','),
       }
     end
   end

data/lib/rails_autoscale_agent/report.rb

@@ -19,14 +19,12 @@ module RailsAutoscaleAgent
     def to_csv
       String.new.tap do |result|
         @measurements.each do |measurement|
-          result << measurement.time.to_i.to_s
-          result << ','
-          result << measurement.value.to_s
-
-          if measurement.queue_name
-            result << ','
-            result << measurement.queue_name
-          end
+          result << [
+            measurement.time.to_i,
+            measurement.value,
+            measurement.queue_name,
+            measurement.metric,
+          ].join(',')
 
           result << "\n"
         end

data/lib/rails_autoscale_agent/reporter.rb

@@ -21,14 +21,14 @@ module RailsAutoscaleAgent
       @started = true
       @worker_adapters = config.worker_adapters.select(&:enabled?)
 
-      if !config.api_base_url
+      if !config.api_base_url && !config.dev_mode?
         logger.info "Reporter not started: #{config.addon_name}_URL is not set"
         return
       end
 
       Thread.new do
         loop do
-          register!(config) unless @registered
+          register!(config, @worker_adapters) unless @registered
 
           # Stagger reporting to spread out reports from many processes
           multiplier = 1 - (rand / 4) # between 0.75 and 1.0
@@ -41,7 +41,7 @@ module RailsAutoscaleAgent
             # Exceptions in threads other than the main thread will fail silently
             # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
             logger.error "Reporter error: #{ex.inspect}"
-            logger.error ex.backtrace.join("\n")
+            AutoscaleApi.new(config.api_base_url).report_exception!(ex)
           end
         end
       end
@@ -58,7 +58,7 @@ module RailsAutoscaleAgent
         logger.info "Reporting #{report.measurements.size} measurements"
 
         params = report.to_params(config)
-        result = AutoscaleApi.new(config.api_base_url).report_metrics!(params, report.to_csv)
+        result = AutoscaleApi.new(config).report_metrics!(params, report.to_csv)
 
         case result
         when AutoscaleApi::SuccessResponse
@@ -71,16 +71,16 @@ module RailsAutoscaleAgent
       end
     end
 
-    def register!(config)
-      params = Registration.new(config).to_params
-      result = AutoscaleApi.new(config.api_base_url).register_reporter!(params)
+    def register!(config, worker_adapters)
+      params = Registration.new(config, worker_adapters).to_params
+      result = AutoscaleApi.new(config).register_reporter!(params)
 
       case result
       when AutoscaleApi::SuccessResponse
         @registered = true
         config.report_interval = result.data['report_interval'] if result.data['report_interval']
         config.max_request_size = result.data['max_request_size'] if result.data['max_request_size']
-        worker_adapters_msg = @worker_adapters.map { |a| a.class.name }.join(', ')
+        worker_adapters_msg = worker_adapters.map { |a| a.class.name }.join(', ')
         logger.info "Reporter starting, will report every #{config.report_interval} seconds or so. Worker adapters: [#{worker_adapters_msg}]"
       when AutoscaleApi::FailureResponse
         logger.error "Reporter failed to register: #{result.failure_message}"

data/lib/rails_autoscale_agent/request.rb

@@ -4,17 +4,17 @@ module RailsAutoscaleAgent
   class Request
     include Logger
 
-    attr_reader :id, :entered_queue_at, :path, :method, :size
-
     def initialize(env, config)
       @config = config
       @id = env['HTTP_X_REQUEST_ID']
-      @path = env['PATH_INFO']
-      @method = env['REQUEST_METHOD'].downcase
       @size = env['rack.input'].respond_to?(:size) ? env['rack.input'].size : 0
+      @request_body_wait = env['puma.request_body_wait'].to_i
 
-      if unix_millis = env['HTTP_X_REQUEST_START']
-        @entered_queue_at = Time.at(unix_millis.to_f / 1000)
+      @entered_queue_at = if unix_millis = env['HTTP_X_REQUEST_START']
+        Time.at(unix_millis.to_f / 1000)
+      elsif config.dev_mode?
+        # In dev mode, fake a queue time of 0-1000ms
+        Time.now - rand + @request_body_wait
       end
     end
 
@@ -23,12 +23,17 @@ module RailsAutoscaleAgent
     end
 
     def queue_time
-      if entered_queue_at
-        queue_time = ((Time.now - entered_queue_at) * 1000).to_i
-        queue_time = 0 if queue_time < 0
-        logger.debug "Collected queue_time=#{queue_time}ms request_id=#{id} request_size=#{size}"
+      if @entered_queue_at
+        queue_time = ((Time.now - @entered_queue_at) * 1000).to_i
+
+        # Subtract the time Puma spent waiting on the request body. It's irrelevant to capacity-related queue time.
+        # Without this, slow clients and large request payloads will skew queue time.
+        queue_time -= @request_body_wait
+
+        logger.debug "Request queue_time=#{queue_time}ms body_wait=#{@request_body_wait}ms request_id=#{@id} size=#{@size}"
 
-        queue_time
+        # Safeguard against negative queue times (should not happen in practice)
+        queue_time > 0 ? queue_time : 0
       end
     end
   end

data/lib/rails_autoscale_agent/store.rb

@@ -15,8 +15,8 @@ module RailsAutoscaleAgent
       @measurements = []
     end
 
-    def push(value, time = Time.now, queue_name = nil)
-      @measurements << Measurement.new(time, value, queue_name)
+    def push(value, time = Time.now, queue_name = nil, metric = nil)
+      @measurements << Measurement.new(time, value, queue_name, metric)
     end
 
     def pop_report

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.8.1"
+  VERSION = "0.9.0.beta.4"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -2,61 +2,69 @@
 
 require 'rails_autoscale_agent/logger'
 
-module WorkerAdapters
-  class DelayedJob
-    include RailsAutoscaleAgent::Logger
-    include Singleton
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class DelayedJob
+      include RailsAutoscaleAgent::Logger
+      include Singleton
 
-    class << self
-      attr_accessor :queues
-    end
+      UNNAMED_QUEUE = '[unnamed]'
 
-    def initialize
-      # Track the known queues so we can continue reporting on queues that don't
-      # currently have enqueued jobs.
-      self.class.queues = Set.new
+      class << self
+        attr_accessor :queues
+      end
 
-      install if enabled?
-    end
+      def initialize
+        # Track the known queues so we can continue reporting on queues that don't
+        # currently have enqueued jobs.
+        self.class.queues = Set.new
 
-    def enabled?
-      defined? ::Delayed
-    end
+        install if enabled?
+      end
 
-    def collect!(store)
-      log_msg = String.new('DelayedJob latency ')
-      t = Time.now
+      def enabled?
+        defined? ::Delayed
+      end
 
-      sql = 'SELECT queue, min(run_at) FROM delayed_jobs GROUP BY queue'
-      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-      queues = self.class.queues | run_at_by_queue.keys
+      def collect!(store)
+        log_msg = String.new
+        t = Time.now.utc
+        db_time = t.strftime('%Y-%m-%d %H:%M:%S')
 
-      queues.each do |queue|
-        run_at = run_at_by_queue[queue]
-        run_at = Time.parse(run_at) if run_at.is_a?(String)
-        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
-        store.push latency_ms, t, queue
-        log_msg << "#{queue}=#{latency_ms} "
-      end
+        # Ignore failed jobs (they skew latency measurement due to the original run_at)
+        sql = "SELECT queue, min(run_at) FROM delayed_jobs WHERE run_at < '#{db_time}' AND locked_at IS NULL AND failed_at IS NULL GROUP BY queue"
+        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+        queues = self.class.queues | run_at_by_queue.keys
 
-      logger.debug log_msg
-    end
+        queues.each do |queue|
+          run_at = run_at_by_queue[queue]
+          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+
+          queue = UNNAMED_QUEUE if queue.nil? || queue.empty?
+          store.push latency_ms, t, queue
+          log_msg << "dj.#{queue}=#{latency_ms} "
+        end
 
-    private
+        logger.debug log_msg unless log_msg.empty?
+      end
 
-    def install
-      plugin = Class.new(Delayed::Plugin) do
-        require 'delayed_job'
+      private
 
-        callbacks do |lifecycle|
-          lifecycle.before(:enqueue) do |job, &block|
-            queue = job.queue || 'default'
-            WorkerAdapters::DelayedJob.queues.add queue
+      def install
+        plugin = Class.new(Delayed::Plugin) do
+          require 'delayed_job'
+
+          callbacks do |lifecycle|
+            lifecycle.before(:enqueue) do |job, &block|
+              queue = job.queue || 'default'
+              WorkerAdapters::DelayedJob.queues.add queue
+            end
           end
         end
-      end
 
-      Delayed::Worker.plugins << plugin
+        Delayed::Worker.plugins << plugin
+      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -3,43 +3,48 @@
 require 'rails_autoscale_agent/logger'
 require 'time'
 
-module WorkerAdapters
-  class Que
-    include RailsAutoscaleAgent::Logger
-    include Singleton
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Que
+      include RailsAutoscaleAgent::Logger
+      include Singleton
 
-    DEFAULT_QUEUES = ['default']
+      UNNAMED_QUEUE = '[unnamed]'
+      DEFAULT_QUEUES = ['default']
 
-    class << self
-      attr_accessor :queues
-    end
-
-    def initialize
-      self.class.queues = DEFAULT_QUEUES
-    end
+      class << self
+        attr_accessor :queues
+      end
 
-    def enabled?
-      defined? ::Que
-    end
+      def initialize
+        self.class.queues = DEFAULT_QUEUES
+      end
 
-    def collect!(store)
-      log_msg = String.new('Que latency ')
-      t = Time.now
-
-      # Ignore failed jobs (they skew latency measurement due to the original run_at)
-      sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
-      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-      self.class.queues |= run_at_by_queue.keys
-
-      self.class.queues.each do |queue|
-        run_at = run_at_by_queue[queue]
-        run_at = Time.parse(run_at) if run_at.is_a?(String)
-        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
-        store.push latency_ms, t, queue
-        log_msg << "#{queue}=#{latency_ms} "
+      def enabled?
+        defined? ::Que
       end
 
-      logger.debug log_msg
+      def collect!(store)
+        log_msg = String.new
+        t = Time.now
+
+        # Ignore failed jobs (they skew latency measurement due to the original run_at)
+        sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
+        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+        self.class.queues |= run_at_by_queue.keys
+
+        self.class.queues.each do |queue|
+          run_at = run_at_by_queue[queue]
+          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+
+          queue = UNNAMED_QUEUE if queue.nil? || queue.empty?
+          store.push latency_ms, t, queue
+          log_msg << "que.#{queue}=#{latency_ms} "
+        end
+
+        logger.debug log_msg unless log_msg.empty?
+      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/resque.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Resque
+      include RailsAutoscaleAgent::Logger
+      include Singleton
+
+      def enabled?
+        require 'resque'
+        true
+      rescue LoadError
+        false
+      end
+
+      def collect!(store)
+        log_msg = String.new
+
+        ::Resque.queues.each do |queue|
+          next if queue.nil? || queue.empty?
+          depth = ::Resque.size(queue)
+          store.push depth, Time.now, queue, :qd
+          log_msg << "resque-qd.#{queue}=#{depth} "
+        end
+
+        logger.debug log_msg
+      end
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -2,28 +2,32 @@
 
 require 'rails_autoscale_agent/logger'
 
-module WorkerAdapters
-  class Sidekiq
-    include RailsAutoscaleAgent::Logger
-    include Singleton
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Sidekiq
+      include RailsAutoscaleAgent::Logger
+      include Singleton
 
-    def enabled?
-      require 'sidekiq/api'
-      true
-    rescue LoadError
-      false
-    end
+      def enabled?
+        require 'sidekiq/api'
+        true
+      rescue LoadError
+        false
+      end
 
-    def collect!(store)
-      log_msg = String.new('Sidekiq latency ')
+      def collect!(store)
+        log_msg = String.new
 
-      ::Sidekiq::Queue.all.each do |queue|
-        latency_ms = (queue.latency * 1000).ceil
-        store.push latency_ms, Time.now, queue.name
-        log_msg << "#{queue.name}=#{latency_ms} "
-      end
+        ::Sidekiq::Queue.all.each do |queue|
+          latency_ms = (queue.latency * 1000).ceil
+          depth = queue.size
+          store.push latency_ms, Time.now, queue.name, :qt
+          store.push depth, Time.now, queue.name, :qd
+          log_msg << "sidekiq-qt.#{queue.name}=#{latency_ms} sidekiq-qd.#{queue.name}=#{depth} "
+        end
 
-      logger.debug log_msg
+        logger.debug log_msg
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0.beta.4
 platform: ruby
 authors:
 - Adam McCrea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-04 00:00:00.000000000 Z
+date: 2020-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -198,6 +198,7 @@ files:
 - lib/rails_autoscale_agent/version.rb
 - lib/rails_autoscale_agent/worker_adapters/delayed_job.rb
 - lib/rails_autoscale_agent/worker_adapters/que.rb
+- lib/rails_autoscale_agent/worker_adapters/resque.rb
 - lib/rails_autoscale_agent/worker_adapters/sidekiq.rb
 - log/.gitkeep
 - rails_autoscale_agent.gemspec
@@ -216,11 +217,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: This gem works with the Rails Autoscale Heroku add-on to automatically scale