rails_autoscale_agent 0.8.3 → 0.9.0.beta.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3924fc840320be8327c183e1cb54055d8516d2fefee1cffbfa7092da39b05985
-  data.tar.gz: abd81e6c4c2dc3a65be7c426dda8742ec92b7c29c47a71eeb2566731436ce4e6
+  metadata.gz: 37ecb709c866b3d831f79d8a15bd54cfcacfd00dc45ebf84ad5129c0d1206a3d
+  data.tar.gz: 373474c8d84bb5cbd026f792aef4eef4028d77b666d74ac4407cd7d4c603703d
 SHA512:
-  metadata.gz: 4867868c9e9e707d9e4afe46f20d19f600a87b26aabf5a96f4ad8ff91698d00b08f8a0566ef20fe8f3f1c036c9220b1034a92c70280a4cf29a6d5b5a728b4105
-  data.tar.gz: e2fbbc4afa90c0c636dddb3172d1164108e984f4c550739df552cd185a8a51d7255ff354debc658ecfa893fb4f87f53e6c298bfe4dddf2aa35615f7636e64420
+  metadata.gz: cbe7b97077c907d45deba6cfa0dce814e8d5fff2339cdf8c81c7e96101d0e3cd8a1dcbcdd3474bcb8fe798ec7591f59b040a8e40d367fa11b32c4dbd6ead52f1
+  data.tar.gz: 2677393fb16bb0fb776c2325838074ed3c0c9c4d990d296e950ad83576456ac66b15765a96e84a1fefe26ff6696ca89e419e23db78a644e01054378c7dbf3cd1

data/README.md

@@ -74,7 +74,7 @@ Reach out to help@railsautoscale.com if you run into any other problems.
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, commit it, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/lib/rails_autoscale_agent/autoscale_api.rb

@@ -11,8 +11,8 @@ module RailsAutoscaleAgent
 
     SUCCESS = 'success'
 
-    def initialize(api_url_base)
-      @api_url_base = api_url_base
+    def initialize(config)
+      @config = config
     end
 
     def report_metrics!(report_params, timings_csv)
@@ -37,9 +37,14 @@ module RailsAutoscaleAgent
     end
 
     def post_raw(options)
-      uri = URI.parse("#{@api_url_base}#{options.fetch(:path)}")
+      uri = URI.parse("#{@config.api_base_url}#{options.fetch(:path)}")
       ssl = uri.scheme == 'https'
 
+      if @config.dev_mode
+        logger.debug "[DEV_MODE] Skipping request to #{uri}"
+        return SuccessResponse.new('{}')
+      end
+
       response = Net::HTTP.start(uri.host, uri.port, use_ssl: ssl) do |http|
         request = Net::HTTP::Post.new(uri.request_uri, options[:headers] || {})
         request.body = options.fetch(:body)

data/lib/rails_autoscale_agent/config.rb

@@ -7,26 +7,29 @@ module RailsAutoscaleAgent
     include Singleton
 
     attr_accessor :report_interval, :logger, :api_base_url, :max_request_size,
-                  :dyno, :pid, :addon_name, :worker_adapters
+                  :dyno, :pid, :addon_name, :worker_adapters, :dev_mode
 
     def initialize
       require 'rails_autoscale_agent/worker_adapters/sidekiq'
       require 'rails_autoscale_agent/worker_adapters/delayed_job'
       require 'rails_autoscale_agent/worker_adapters/que'
+      require 'rails_autoscale_agent/worker_adapters/resque'
       @worker_adapters = [
         WorkerAdapters::Sidekiq.instance,
         WorkerAdapters::DelayedJob.instance,
         WorkerAdapters::Que.instance,
+        WorkerAdapters::Resque.instance,
       ]
 
       # Allow the add-on name to be configured - needed for testing
       @addon_name = ENV['RAILS_AUTOSCALE_ADDON'] || 'RAILS_AUTOSCALE'
       @api_base_url = ENV["#{@addon_name}_URL"]
+      @dev_mode = ENV['RAILS_AUTOSCALE_DEV'] == 'true'
       @pid = Process.pid
       @max_request_size = 100_000 # ignore request payloads over 100k since they skew the queue times
-      @report_interval = 60 # this default will be overwritten during Reporter#register!
+      @report_interval = 10 # this default will be overwritten during Reporter#register!
       @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
-      @dyno = ENV['DYNO']
+      @dyno = @dev_mode ? 'dev.1' : ENV['DYNO']
     end
 
     def to_s
@@ -37,5 +40,6 @@ module RailsAutoscaleAgent
       @max_request_size
     end
 
+    alias_method :dev_mode?, :dev_mode
   end
 end

data/lib/rails_autoscale_agent/logger.rb

@@ -24,7 +24,7 @@ module RailsAutoscaleAgent
       # to DEBUG level in production).
       # This uses a separate logger so that RAILS_AUTOSCALE_DEBUG
       # shows debug logs regardless of Rails log level.
-      debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true'
+      debug_logger.debug tag(msg) if ENV['RAILS_AUTOSCALE_DEBUG'] == 'true' || Config.instance.dev_mode?
     end
 
     private

data/lib/rails_autoscale_agent/measurement.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  class Measurement < Struct.new(:time, :value, :queue_name)
-    def initialize(time, value, queue_name = nil)
-      super time.utc, value.to_i, queue_name
+  class Measurement < Struct.new(:time, :value, :queue_name, :metric)
+    # No queue_name is assumed to be a web request measurement
+    # Metrics: qt = queue time (default), qd = queue depth (needed for Resque support)
+    def initialize(time, value, queue_name = nil, metric = nil)
+      super time.utc, value.to_i, queue_name, metric
     end
   end
 end

data/lib/rails_autoscale_agent/registration.rb

@@ -3,7 +3,7 @@
 require 'rails_autoscale_agent/version'
 
 module RailsAutoscaleAgent
-  class Registration < Struct.new(:config)
+  class Registration < Struct.new(:config, :worker_adapters)
 
     def to_params
       {
@@ -12,6 +12,8 @@ module RailsAutoscaleAgent
         ruby_version: RUBY_VERSION,
         rails_version: defined?(Rails) && Rails.version,
         gem_version: VERSION,
+        # example: { worker_adapters: 'Sidekiq,Que' }
+        worker_adapters: worker_adapters.map { |o| o.class.name.split('::').last }.join(','),
       }
     end
   end

data/lib/rails_autoscale_agent/report.rb

@@ -19,14 +19,12 @@ module RailsAutoscaleAgent
     def to_csv
       String.new.tap do |result|
         @measurements.each do |measurement|
-          result << measurement.time.to_i.to_s
-          result << ','
-          result << measurement.value.to_s
-
-          if measurement.queue_name
-            result << ','
-            result << measurement.queue_name
-          end
+          result << [
+            measurement.time.to_i,
+            measurement.value,
+            measurement.queue_name,
+            measurement.metric,
+          ].join(',')
 
           result << "\n"
         end

data/lib/rails_autoscale_agent/reporter.rb

@@ -21,14 +21,14 @@ module RailsAutoscaleAgent
       @started = true
       @worker_adapters = config.worker_adapters.select(&:enabled?)
 
-      if !config.api_base_url
+      if !config.api_base_url && !config.dev_mode?
         logger.info "Reporter not started: #{config.addon_name}_URL is not set"
         return
       end
 
       Thread.new do
         loop do
-          register!(config) unless @registered
+          register!(config, @worker_adapters) unless @registered
 
           # Stagger reporting to spread out reports from many processes
           multiplier = 1 - (rand / 4) # between 0.75 and 1.0
@@ -58,7 +58,7 @@ module RailsAutoscaleAgent
         logger.info "Reporting #{report.measurements.size} measurements"
 
         params = report.to_params(config)
-        result = AutoscaleApi.new(config.api_base_url).report_metrics!(params, report.to_csv)
+        result = AutoscaleApi.new(config).report_metrics!(params, report.to_csv)
 
         case result
         when AutoscaleApi::SuccessResponse
@@ -71,16 +71,16 @@ module RailsAutoscaleAgent
       end
     end
 
-    def register!(config)
-      params = Registration.new(config).to_params
-      result = AutoscaleApi.new(config.api_base_url).register_reporter!(params)
+    def register!(config, worker_adapters)
+      params = Registration.new(config, worker_adapters).to_params
+      result = AutoscaleApi.new(config).register_reporter!(params)
 
       case result
       when AutoscaleApi::SuccessResponse
         @registered = true
         config.report_interval = result.data['report_interval'] if result.data['report_interval']
         config.max_request_size = result.data['max_request_size'] if result.data['max_request_size']
-        worker_adapters_msg = @worker_adapters.map { |a| a.class.name }.join(', ')
+        worker_adapters_msg = worker_adapters.map { |a| a.class.name }.join(', ')
         logger.info "Reporter starting, will report every #{config.report_interval} seconds or so. Worker adapters: [#{worker_adapters_msg}]"
       when AutoscaleApi::FailureResponse
         logger.error "Reporter failed to register: #{result.failure_message}"

data/lib/rails_autoscale_agent/request.rb

@@ -13,8 +13,11 @@ module RailsAutoscaleAgent
       @method = env['REQUEST_METHOD'].downcase
       @size = env['rack.input'].respond_to?(:size) ? env['rack.input'].size : 0
 
-      if unix_millis = env['HTTP_X_REQUEST_START']
-        @entered_queue_at = Time.at(unix_millis.to_f / 1000)
+      @entered_queue_at = if unix_millis = env['HTTP_X_REQUEST_START']
+        Time.at(unix_millis.to_f / 1000)
+      elsif config.dev_mode?
+        # In dev mode, fake a queue time of 0-1000ms
+        Time.now - rand
       end
     end
 

data/lib/rails_autoscale_agent/store.rb

@@ -15,8 +15,8 @@ module RailsAutoscaleAgent
       @measurements = []
     end
 
-    def push(value, time = Time.now, queue_name = nil)
-      @measurements << Measurement.new(time, value, queue_name)
+    def push(value, time = Time.now, queue_name = nil, metric = nil)
+      @measurements << Measurement.new(time, value, queue_name, metric)
     end
 
     def pop_report

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.8.3"
+  VERSION = "0.9.0.beta.1"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -2,63 +2,64 @@
 
 require 'rails_autoscale_agent/logger'
 
-module WorkerAdapters
-  class DelayedJob
-    include RailsAutoscaleAgent::Logger
-    include Singleton
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class DelayedJob
+      include RailsAutoscaleAgent::Logger
+      include Singleton
 
-    class << self
-      attr_accessor :queues
-    end
+      class << self
+        attr_accessor :queues
+      end
 
-    def initialize
-      # Track the known queues so we can continue reporting on queues that don't
-      # currently have enqueued jobs.
-      self.class.queues = Set.new
+      def initialize
+        # Track the known queues so we can continue reporting on queues that don't
+        # currently have enqueued jobs.
+        self.class.queues = Set.new
 
-      install if enabled?
-    end
+        install if enabled?
+      end
 
-    def enabled?
-      defined? ::Delayed
-    end
+      def enabled?
+        defined? ::Delayed
+      end
 
-    def collect!(store)
-      log_msg = String.new('DelayedJob latency ')
-      t = Time.now
+      def collect!(store)
+        log_msg = String.new
+        t = Time.now
 
-      # Ignore failed jobs (they skew latency measurement due to the original run_at)
-      sql = 'SELECT queue, min(run_at) FROM delayed_jobs WHERE attempts = 0 GROUP BY queue'
-      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-      queues = self.class.queues | run_at_by_queue.keys
+        sql = 'SELECT queue, min(run_at) FROM delayed_jobs GROUP BY queue'
+        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+        queues = self.class.queues | run_at_by_queue.keys
 
-      queues.each do |queue|
-        next if queue.nil? || queue.empty?
-        run_at = run_at_by_queue[queue]
-        run_at = Time.parse(run_at) if run_at.is_a?(String)
-        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
-        store.push latency_ms, t, queue
-        log_msg << "#{queue}=#{latency_ms} "
-      end
+        queues.each do |queue|
+          next if queue.nil? || queue.empty?
+          run_at = run_at_by_queue[queue]
+          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          store.push latency_ms, t, queue
+          log_msg << "dj.#{queue}=#{latency_ms} "
+        end
 
-      logger.debug log_msg
-    end
+        logger.debug log_msg unless log_msg.empty?
+      end
 
-    private
+      private
 
-    def install
-      plugin = Class.new(Delayed::Plugin) do
-        require 'delayed_job'
+      def install
+        plugin = Class.new(Delayed::Plugin) do
+          require 'delayed_job'
 
-        callbacks do |lifecycle|
-          lifecycle.before(:enqueue) do |job, &block|
-            queue = job.queue || 'default'
-            WorkerAdapters::DelayedJob.queues.add queue
+          callbacks do |lifecycle|
+            lifecycle.before(:enqueue) do |job, &block|
+              queue = job.queue || 'default'
+              WorkerAdapters::DelayedJob.queues.add queue
+            end
           end
         end
-      end
 
-      Delayed::Worker.plugins << plugin
+        Delayed::Worker.plugins << plugin
+      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -3,44 +3,46 @@
 require 'rails_autoscale_agent/logger'
 require 'time'
 
-module WorkerAdapters
-  class Que
-    include RailsAutoscaleAgent::Logger
-    include Singleton
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Que
+      include RailsAutoscaleAgent::Logger
+      include Singleton
 
-    DEFAULT_QUEUES = ['default']
+      DEFAULT_QUEUES = ['default']
 
-    class << self
-      attr_accessor :queues
-    end
-
-    def initialize
-      self.class.queues = DEFAULT_QUEUES
-    end
+      class << self
+        attr_accessor :queues
+      end
 
-    def enabled?
-      defined? ::Que
-    end
+      def initialize
+        self.class.queues = DEFAULT_QUEUES
+      end
 
-    def collect!(store)
-      log_msg = String.new('Que latency ')
-      t = Time.now
-
-      # Ignore failed jobs (they skew latency measurement due to the original run_at)
-      sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
-      run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-      self.class.queues |= run_at_by_queue.keys
-
-      self.class.queues.each do |queue|
-        next if queue.nil? || queue.empty?
-        run_at = run_at_by_queue[queue]
-        run_at = Time.parse(run_at) if run_at.is_a?(String)
-        latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
-        store.push latency_ms, t, queue
-        log_msg << "#{queue}=#{latency_ms} "
+      def enabled?
+        defined? ::Que
       end
 
-      logger.debug log_msg
+      def collect!(store)
+        log_msg = String.new
+        t = Time.now
+
+        # Ignore failed jobs (they skew latency measurement due to the original run_at)
+        sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
+        run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
+        self.class.queues |= run_at_by_queue.keys
+
+        self.class.queues.each do |queue|
+          next if queue.nil? || queue.empty?
+          run_at = run_at_by_queue[queue]
+          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          store.push latency_ms, t, queue
+          log_msg << "que.#{queue}=#{latency_ms} "
+        end
+
+        logger.debug log_msg unless log_msg.empty?
+      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/resque.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Resque
+      include RailsAutoscaleAgent::Logger
+      include Singleton
+
+      def enabled?
+        require 'resque'
+        true
+      rescue LoadError
+        false
+      end
+
+      def collect!(store)
+        log_msg = String.new
+
+        ::Resque.queues.each do |queue|
+          next if queue.nil? || queue.empty?
+          depth = ::Resque.size(queue)
+          store.push depth, Time.now, queue, :qd
+          log_msg << "resque-qd.#{queue}=#{depth} "
+        end
+
+        logger.debug log_msg
+      end
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -2,28 +2,32 @@
 
 require 'rails_autoscale_agent/logger'
 
-module WorkerAdapters
-  class Sidekiq
-    include RailsAutoscaleAgent::Logger
-    include Singleton
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Sidekiq
+      include RailsAutoscaleAgent::Logger
+      include Singleton
 
-    def enabled?
-      require 'sidekiq/api'
-      true
-    rescue LoadError
-      false
-    end
+      def enabled?
+        require 'sidekiq/api'
+        true
+      rescue LoadError
+        false
+      end
 
-    def collect!(store)
-      log_msg = String.new('Sidekiq latency ')
+      def collect!(store)
+        log_msg = String.new
 
-      ::Sidekiq::Queue.all.each do |queue|
-        latency_ms = (queue.latency * 1000).ceil
-        store.push latency_ms, Time.now, queue.name
-        log_msg << "#{queue.name}=#{latency_ms} "
-      end
+        ::Sidekiq::Queue.all.each do |queue|
+          latency_ms = (queue.latency * 1000).ceil
+          depth = queue.size
+          store.push latency_ms, Time.now, queue.name, :qt
+          store.push depth, Time.now, queue.name, :qd
+          log_msg << "sidekiq-qt.#{queue.name}=#{latency_ms} sidekiq-qd.#{queue.name}=#{depth} "
+        end
 
-      logger.debug log_msg
+        logger.debug log_msg
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.9.0.beta.1
 platform: ruby
 authors:
 - Adam McCrea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-26 00:00:00.000000000 Z
+date: 2020-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -198,6 +198,7 @@ files:
 - lib/rails_autoscale_agent/version.rb
 - lib/rails_autoscale_agent/worker_adapters/delayed_job.rb
 - lib/rails_autoscale_agent/worker_adapters/que.rb
+- lib/rails_autoscale_agent/worker_adapters/resque.rb
 - lib/rails_autoscale_agent/worker_adapters/sidekiq.rb
 - log/.gitkeep
 - rails_autoscale_agent.gemspec
@@ -216,9 +217,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: