rails_autoscale_agent 0.7.0 → 0.10.2

data/lib/rails_autoscale_agent/logger.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext/module/delegation'
 require 'rails_autoscale_agent/config'
+require 'logger'
 
 module RailsAutoscaleAgent
   module Logger
@@ -11,23 +11,37 @@ module RailsAutoscaleAgent
   end
 
   class LoggerProxy < Struct.new(:logger)
-    def tagged(*tags, &block)
-      if logger.respond_to?(:tagged)
-        logger.tagged *tags, &block
-      else
-        # NOTE: Quack like ActiveSupport::TaggedLogging, but don't reimplement
-        yield self
-      end
+    TAG = '[RailsAutoscale]'
+
+    def error(msg)
+      logger.error tag(msg)
+    end
+
+    def warn(msg)
+      logger.warn tag(msg)
     end
 
-    def debug(*args)
-      # Rails logger defaults to DEBUG level in production, but I don't want
-      # to be chatty by default.
-      logger.debug(*args) if ENV['RAILS_AUTOSCALE_LOG_LEVEL'] == 'DEBUG'
+    def info(msg)
+      logger.info tag(msg) unless Config.instance.quiet?
     end
 
-    def method_missing(name, *args, &block)
-      logger.send name, *args, &block
+    def debug(msg)
+      # Silence debug logs by default to avoiding being overly chatty (Rails logger defaults
+      # to DEBUG level in production). Setting RAILS_AUTOSCALE_DEBUG=true enables debug logs,
+      # even if the underlying logger severity level is INFO.
+      if Config.instance.debug?
+        if logger.respond_to?(:debug?) && logger.debug?
+          logger.debug tag(msg)
+        elsif logger.respond_to?(:info?) && logger.info?
+          logger.info tag("[DEBUG] #{msg}")
+        end
+      end
+    end
+
+    private
+
+    def tag(msg)
+      "#{TAG} #{msg}"
     end
   end
 end

data/lib/rails_autoscale_agent/measurement.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  class Measurement < Struct.new(:time, :value, :queue_name)
-    def initialize(time, value, queue_name = nil)
-      super time.utc, value.to_i, queue_name
+  class Measurement < Struct.new(:time, :value, :queue_name, :metric)
+    # No queue_name is assumed to be a web request measurement
+    # Metrics: qt = queue time (default), qd = queue depth (needed for Resque support)
+    def initialize(time, value, queue_name = nil, metric = nil)
+      super time.utc, value.to_i, queue_name, metric
     end
   end
 end

data/lib/rails_autoscale_agent/middleware.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'rails_autoscale_agent/logger'
 require 'rails_autoscale_agent/store'
 require 'rails_autoscale_agent/reporter'
 require 'rails_autoscale_agent/config'
@@ -8,25 +7,21 @@ require 'rails_autoscale_agent/request'
 
 module RailsAutoscaleAgent
   class Middleware
-    include Logger
-
     def initialize(app)
       @app = app
     end
 
     def call(env)
-      logger.tagged 'RailsAutoscale' do
-        config = Config.instance
-        request = Request.new(env, config)
+      config = Config.instance
+      request = Request.new(env, config)
 
-        store = Store.instance
-        Reporter.start(config, store)
+      store = Store.instance
+      Reporter.start(config, store)
 
-        if !request.ignore? && queue_time = request.queue_time
-          # NOTE: Expose queue time to the app
-          env['queue_time'] = queue_time
-          store.push queue_time
-        end
+      if !request.ignore? && queue_time = request.queue_time
+        # NOTE: Expose queue time to the app
+        env['queue_time'] = queue_time
+        store.push queue_time
       end
 
       @app.call(env)

data/lib/rails_autoscale_agent/railtie.rb

@@ -1,10 +1,14 @@
 # frozen_string_literal: true
 
 require 'rails_autoscale_agent/middleware'
+require 'rails_autoscale_agent/logger'
 
 module RailsAutoscaleAgent
   class Railtie < Rails::Railtie
+    include Logger
+
     initializer "rails_autoscale_agent.middleware" do |app|
+      logger.info "Preparing middleware"
       app.middleware.insert_before Rack::Runtime, Middleware
     end
   end

data/lib/rails_autoscale_agent/registration.rb

@@ -3,15 +3,17 @@
 require 'rails_autoscale_agent/version'
 
 module RailsAutoscaleAgent
-  class Registration < Struct.new(:config)
+  class Registration < Struct.new(:config, :worker_adapters)
 
     def to_params
       {
         dyno: config.dyno,
-        pid: config.pid,
+        pid: Process.pid,
         ruby_version: RUBY_VERSION,
         rails_version: defined?(Rails) && Rails.version,
         gem_version: VERSION,
+        # example: { worker_adapters: 'Sidekiq,Que' }
+        worker_adapters: worker_adapters.map { |o| o.class.name.split('::').last }.join(','),
       }
     end
   end

data/lib/rails_autoscale_agent/report.rb

@@ -12,21 +12,19 @@ module RailsAutoscaleAgent
     def to_params(config)
       {
         dyno: config.dyno,
-        pid: config.pid,
+        pid: Process.pid,
       }
     end
 
     def to_csv
       String.new.tap do |result|
         @measurements.each do |measurement|
-          result << measurement.time.to_i.to_s
-          result << ','
-          result << measurement.value.to_s
-
-          if measurement.queue_name
-            result << ','
-            result << measurement.queue_name
-          end
+          result << [
+            measurement.time.to_i,
+            measurement.value,
+            measurement.queue_name,
+            measurement.metric,
+          ].join(',')
 
           result << "\n"
         end

data/lib/rails_autoscale_agent/reporter.rb

@@ -5,7 +5,6 @@ require 'rails_autoscale_agent/logger'
 require 'rails_autoscale_agent/autoscale_api'
 require 'rails_autoscale_agent/time_rounder'
 require 'rails_autoscale_agent/registration'
-require 'rails_autoscale_agent/worker_adapters/sidekiq'
 
 # Reporter wakes up every minute to send metrics to the RailsAutoscale API
 
@@ -21,31 +20,29 @@ module RailsAutoscaleAgent
     def start!(config, store)
       @started = true
       @worker_adapters = config.worker_adapters.select(&:enabled?)
+      @dyno_num = config.dyno.to_s.split('.').last.to_i
 
-      if !config.api_base_url
+      if !config.api_base_url && !config.dev_mode?
         logger.info "Reporter not started: #{config.addon_name}_URL is not set"
         return
       end
 
       Thread.new do
-        logger.tagged 'RailsAutoscale' do
-          register!(config)
-
-          loop do
-            # Stagger reporting to spread out reports from many processes
-            multiplier = 1 - (rand / 4) # between 0.75 and 1.0
-            sleep config.report_interval * multiplier
-
-            begin
-              @worker_adapters.map { |a| a.collect!(store) }
-              report!(config, store)
-            rescue => ex
-              # Exceptions in threads other than the main thread will fail silently
-              # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
-              logger.error "Reporter error: #{ex.inspect}"
-              logger.error ex.backtrace.join("\n")
+        loop do
+          register!(config, @worker_adapters) unless @registered
+
+          # Stagger reporting to spread out reports from many processes
+          multiplier = 1 - (rand / 4) # between 0.75 and 1.0
+          sleep config.report_interval * multiplier
+
+          # It's redundant to report worker metrics from every web dyno, so only report from web.1
+          if @dyno_num == 1
+            @worker_adapters.map do |adapter|
+              report_exceptions(config) { adapter.collect!(store) }
             end
           end
+
+          report_exceptions(config) { report!(config, store) }
         end
       end
     end
@@ -54,6 +51,8 @@ module RailsAutoscaleAgent
       @started
     end
 
+    private
+
     def report!(config, store)
       report = store.pop_report
 
@@ -61,7 +60,7 @@ module RailsAutoscaleAgent
         logger.info "Reporting #{report.measurements.size} measurements"
 
         params = report.to_params(config)
-        result = AutoscaleApi.new(config.api_base_url).report_metrics!(params, report.to_csv)
+        result = AutoscaleApi.new(config).report_metrics!(params, report.to_csv)
 
         case result
         when AutoscaleApi::SuccessResponse
@@ -74,20 +73,35 @@ module RailsAutoscaleAgent
       end
     end
 
-    def register!(config)
-      params = Registration.new(config).to_params
-      result = AutoscaleApi.new(config.api_base_url).register_reporter!(params)
+    def register!(config, worker_adapters)
+      params = Registration.new(config, worker_adapters).to_params
+      result = AutoscaleApi.new(config).register_reporter!(params)
 
       case result
       when AutoscaleApi::SuccessResponse
+        @registered = true
         config.report_interval = result.data['report_interval'] if result.data['report_interval']
         config.max_request_size = result.data['max_request_size'] if result.data['max_request_size']
-        worker_adapters_msg = @worker_adapters.map { |a| a.class.name }.join(', ')
+        worker_adapters_msg = worker_adapters.map { |a| a.class.name }.join(', ')
         logger.info "Reporter starting, will report every #{config.report_interval} seconds or so. Worker adapters: [#{worker_adapters_msg}]"
       when AutoscaleApi::FailureResponse
         logger.error "Reporter failed to register: #{result.failure_message}"
       end
     end
 
+    def report_exceptions(config)
+      begin
+        yield
+      rescue => ex
+        # Exceptions in threads other than the main thread will fail silently
+        # https://ruby-doc.org/core-2.2.0/Thread.html#class-Thread-label-Exception+handling
+        logger.error "Reporter error: #{ex.inspect}"
+        AutoscaleApi.new(config).report_exception!(ex)
+      end
+    rescue => ex
+      # An exception was encountered while trying to report the original exception.
+      # Swallow the error so the reporter continues to report.
+      logger.error "Exception reporting error: #{ex.inspect}"
+    end
   end
 end

data/lib/rails_autoscale_agent/request.rb

@@ -1,35 +1,49 @@
 # frozen_string_literal: true
 
+require 'rails_autoscale_agent/logger'
+
 module RailsAutoscaleAgent
   class Request
     include Logger
 
-    attr_reader :id, :entered_queue_at, :path, :method, :size
-
     def initialize(env, config)
       @config = config
       @id = env['HTTP_X_REQUEST_ID']
-      @path = env['PATH_INFO']
-      @method = env['REQUEST_METHOD'].downcase
       @size = env['rack.input'].respond_to?(:size) ? env['rack.input'].size : 0
-
-      if unix_millis = env['HTTP_X_REQUEST_START']
-        @entered_queue_at = Time.at(unix_millis.to_f / 1000)
-      end
+      @request_body_wait = env['puma.request_body_wait'].to_i
+      @request_start_header = env['HTTP_X_REQUEST_START']
     end
 
     def ignore?
       @config.ignore_large_requests? && @size > @config.max_request_size
     end
 
-    def queue_time
-      if entered_queue_at
-        queue_time = ((Time.now - entered_queue_at) * 1000).to_i
-        queue_time = 0 if queue_time < 0
-        logger.debug "Collected queue_time=#{queue_time}ms request_id=#{id} request_size=#{size}"
-
-        queue_time
+    def started_at
+      if @request_start_header
+        # Heroku sets the header as an integer, measured in milliseconds.
+        # If nginx is involved, it might be in seconds with fractional milliseconds,
+        # and it might be preceeded by "t=". We can all cases by removing non-digits
+        # and treating as milliseconds.
+        Time.at(@request_start_header.gsub(/\D/, '').to_i / 1000.0)
+      elsif @config.dev_mode?
+        # In dev mode, fake a queue time of 0-1000ms
+        Time.now - rand + @request_body_wait
       end
     end
+
+    def queue_time(now = Time.now)
+      return if started_at.nil?
+
+      queue_time = ((now - started_at) * 1000).to_i
+
+      # Subtract the time Puma spent waiting on the request body. It's irrelevant to capacity-related queue time.
+      # Without this, slow clients and large request payloads will skew queue time.
+      queue_time -= @request_body_wait
+
+      logger.debug "Request queue_time=#{queue_time}ms body_wait=#{@request_body_wait}ms request_id=#{@id} size=#{@size}"
+
+      # Safeguard against negative queue times (should not happen in practice)
+      queue_time > 0 ? queue_time : 0
+    end
   end
 end

data/lib/rails_autoscale_agent/store.rb

@@ -15,11 +15,16 @@ module RailsAutoscaleAgent
       @measurements = []
     end
 
-    def push(value, time = Time.now, queue_name = nil)
-      @measurements << Measurement.new(time, value, queue_name)
+    def push(value, time = Time.now, queue_name = nil, metric = nil)
+      # If it's been two minutes since clearing out the store, stop collecting measurements.
+      # There could be an issue with the reporter, and continuing to collect will consume linear memory.
+      return if @last_pop && @last_pop < Time.now - 120
+
+      @measurements << Measurement.new(time, value, queue_name, metric)
     end
 
     def pop_report
+      @last_pop = Time.now
       report = Report.new
 
       while measurement = @measurements.shift

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.7.0"
+  VERSION = "0.10.2"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class DelayedJob
+      include RailsAutoscaleAgent::Logger
+      include Singleton
+
+      attr_writer :queues
+
+      def enabled?
+        if defined?(::Delayed::Job) && defined?(::Delayed::Backend::ActiveRecord)
+          log_msg = String.new("DelayedJob enabled (#{::ActiveRecord::Base.default_timezone})")
+          log_msg << " with long-running job support" if track_long_running_jobs?
+          logger.info log_msg
+          true
+        end
+      end
+
+      def collect!(store)
+        log_msg = String.new
+        t = Time.now.utc
+        sql = <<~SQL
+          SELECT COALESCE(queue, 'default'), min(run_at)
+          FROM delayed_jobs
+          WHERE locked_at IS NULL
+          AND failed_at IS NULL
+          GROUP BY queue
+        SQL
+
+        run_at_by_queue = Hash[select_rows(sql)]
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if run_at_by_queue.size > 50
+          logger.debug "Skipping DelayedJob metrics - #{run_at_by_queue.size} queues"
+          return
+        end
+
+        self.queues = queues | run_at_by_queue.keys
+
+        if track_long_running_jobs?
+          sql = <<~SQL
+            SELECT COALESCE(queue, 'default'), count(*)
+            FROM delayed_jobs
+            WHERE locked_at IS NOT NULL
+            AND locked_by IS NOT NULL
+            AND failed_at IS NULL
+            GROUP BY 1
+          SQL
+
+          busy_count_by_queue = Hash[select_rows(sql)]
+          self.queues = queues | busy_count_by_queue.keys
+        end
+
+        queues.each do |queue|
+          run_at = run_at_by_queue[queue]
+          # DateTime.parse assumes a UTC string
+          run_at = DateTime.parse(run_at) if run_at.is_a?(String)
+          latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          latency_ms = 0 if latency_ms < 0
+
+          store.push latency_ms, t, queue
+          log_msg << "dj-qt.#{queue}=#{latency_ms} "
+
+          if track_long_running_jobs?
+            busy_count = busy_count_by_queue[queue] || 0
+            store.push busy_count, Time.now, queue, :busy
+            log_msg << "dj-busy.#{queue}=#{busy_count} "
+          end
+        end
+
+        logger.debug log_msg unless log_msg.empty?
+      end
+
+      private
+
+      def queues
+        # Track the known queues so we can continue reporting on queues that don't
+        # have enqueued jobs at the time of reporting.
+        # Assume a "default" queue so we always report *something*, even when nothing
+        # is enqueued.
+        @queues ||= Set.new(['default'])
+      end
+
+      def track_long_running_jobs?
+        Config.instance.track_long_running_jobs
+      end
+
+      def select_rows(sql)
+        # This ensures the agent doesn't hold onto a DB connection any longer than necessary
+        ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
+      end
+    end
+  end
+end