rails_autoscale_agent 0.7.0 → 0.10.2

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Que
+      include RailsAutoscaleAgent::Logger
+      include Singleton
+
+      attr_writer :queues
+
+      def queues
+        # Track the known queues so we can continue reporting on queues that don't
+        # have enqueued jobs at the time of reporting.
+        # Assume a "default" queue so we always report *something*, even when nothing
+        # is enqueued.
+        @queues ||= Set.new(['default'])
+      end
+
+      def enabled?
+        if defined?(::Que)
+          logger.info "Que enabled (#{::ActiveRecord::Base.default_timezone})"
+          true
+        end
+      end
+
+      def collect!(store)
+        log_msg = String.new
+        t = Time.now.utc
+        sql = <<~SQL
+          SELECT queue, min(run_at)
+          FROM que_jobs
+          WHERE finished_at IS NULL
+          AND expired_at IS NULL
+          AND error_count = 0
+          GROUP BY 1
+        SQL
+
+        run_at_by_queue = Hash[select_rows(sql)]
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if run_at_by_queue.size > 50
+          logger.debug "Skipping Que metrics - #{run_at_by_queue.size} queues"
+          return
+        end
+
+        self.queues |= run_at_by_queue.keys
+
+        queues.each do |queue|
+          run_at = run_at_by_queue[queue]
+          run_at = DateTime.parse(run_at) if run_at.is_a?(String)
+          latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          latency_ms = 0 if latency_ms < 0
+
+          store.push latency_ms, t, queue
+          log_msg << "que.#{queue}=#{latency_ms} "
+        end
+
+        logger.debug log_msg unless log_msg.empty?
+      end
+
+      private
+
+      def select_rows(sql)
+        # This ensures the agent doesn't hold onto a DB connection any longer than necessary
+        ActiveRecord::Base.connection_pool.with_connection { |c| c.select_rows(sql) }
+      end
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/resque.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rails_autoscale_agent/logger'
+
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Resque
+      include RailsAutoscaleAgent::Logger
+      include Singleton
+
+      attr_writer :queues
+
+      def queues
+        @queues ||= ['default']
+      end
+
+      def enabled?
+        require 'resque'
+        logger.info "Resque enabled"
+        true
+      rescue LoadError
+        false
+      end
+
+      def collect!(store)
+        log_msg = String.new
+        current_queues = ::Resque.queues
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if current_queues.size > 50
+          logger.debug "Skipping Resque metrics - #{current_queues.size} queues"
+          return
+        end
+
+        # Ensure we continue to collect metrics for known queue names, even when nothing is
+        # enqueued at the time. Without this, it will appears that the agent is no longer reporting.
+        self.queues |= current_queues
+
+        queues.each do |queue|
+          next if queue.nil? || queue.empty?
+          depth = ::Resque.size(queue)
+          store.push depth, Time.now, queue, :qd
+          log_msg << "resque-qd.#{queue}=#{depth} "
+        end
+
+        logger.debug log_msg
+      end
+    end
+  end
+end

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -2,28 +2,79 @@
 
 require 'rails_autoscale_agent/logger'
 
-module WorkerAdapters
-  class Sidekiq
-    include RailsAutoscaleAgent::Logger
-
-    def enabled?
-      require 'sidekiq/api'
-      true
-    rescue LoadError
-      false
-    end
+module RailsAutoscaleAgent
+  module WorkerAdapters
+    class Sidekiq
+      include RailsAutoscaleAgent::Logger
+      include Singleton
+
+      attr_writer :queues
+
+      def enabled?
+        require 'sidekiq/api'
+
+        log_msg = String.new("Sidekiq enabled")
+        log_msg << " with long-running job support" if track_long_running_jobs?
+        logger.info log_msg
+
+        true
+      rescue LoadError
+        false
+      end
+
+      def collect!(store)
+        log_msg = String.new
+        queues_by_name = ::Sidekiq::Queue.all.each_with_object({}) do |queue, obj|
+          obj[queue.name] = queue
+        end
+
+        # Don't collect worker metrics if there are unreasonable number of queues
+        if queues_by_name.size > 50
+          logger.debug "Skipping Sidekiq metrics - #{queues_by_name.size} queues"
+          return
+        end
 
-    # TODO: specs
-    def collect!(store)
-      log_msg = String.new('Sidekiq latency ')
+        # Ensure we continue to collect metrics for known queue names, even when nothing is
+        # enqueued at the time. Without this, it will appear that the agent is no longer reporting.
+        queues.each do |queue_name|
+          queues_by_name[queue_name] ||= ::Sidekiq::Queue.new(queue_name)
+        end
+        self.queues = queues_by_name.keys
 
-      ::Sidekiq::Queue.all.each do |queue|
-        latency_ms = (queue.latency * 1000).ceil
-        store.push latency_ms, Time.now, queue.name
-        log_msg << "#{queue.name}=#{latency_ms} "
+        if track_long_running_jobs?
+          busy_counts = Hash.new { |h,k| h[k] = 0}
+          ::Sidekiq::Workers.new.each do |pid, tid, work|
+            busy_counts[work.dig('payload', 'queue')] += 1
+          end
+        end
+
+        queues_by_name.each do |queue_name, queue|
+          latency_ms = (queue.latency * 1000).ceil
+          depth = queue.size
+
+          store.push latency_ms, Time.now, queue.name, :qt
+          store.push depth, Time.now, queue.name, :qd
+          log_msg << "sidekiq-qt.#{queue.name}=#{latency_ms} sidekiq-qd.#{queue.name}=#{depth} "
+
+          if track_long_running_jobs?
+            busy_count = busy_counts[queue.name]
+            store.push busy_count, Time.now, queue.name, :busy
+            log_msg << "sidekiq-busy.#{queue.name}=#{busy_count} "
+          end
+        end
+
+        logger.debug log_msg
       end
 
-      logger.debug log_msg
+      private
+
+      def queues
+        @queues ||= ['default']
+      end
+
+      def track_long_running_jobs?
+        Config.instance.track_long_running_jobs
+      end
     end
   end
 end

data/rails_autoscale_agent.gemspec

@@ -16,14 +16,5 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport", ">= 3.2"
-
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "vcr", "~> 3.0"
-  spec.add_development_dependency "webmock"
-  spec.add_development_dependency "pry"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "sidekiq", "~> 5.0"
+  spec.required_ruby_version = '>= 2.5'
 end

metadata

@@ -1,142 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.10.2
 platform: ruby
 authors:
 - Adam McCrea
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-04 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-description: 
+date: 2021-01-12 00:00:00.000000000 Z
+dependencies: []
+description:
 email:
 - adam@adamlogic.com
 executables: []
@@ -147,6 +21,8 @@ files:
 - ".rspec"
 - ".ruby-version"
 - ".travis.yml"
+- ".vscode/tasks.json"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -167,6 +43,9 @@ files:
 - lib/rails_autoscale_agent/store.rb
 - lib/rails_autoscale_agent/time_rounder.rb
 - lib/rails_autoscale_agent/version.rb
+- lib/rails_autoscale_agent/worker_adapters/delayed_job.rb
+- lib/rails_autoscale_agent/worker_adapters/que.rb
+- lib/rails_autoscale_agent/worker_adapters/resque.rb
 - lib/rails_autoscale_agent/worker_adapters/sidekiq.rb
 - log/.gitkeep
 - rails_autoscale_agent.gemspec
@@ -174,7 +53,7 @@ homepage: https://github.com/adamlogic/rails_autoscale_agent
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -182,15 +61,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: This gem works with the Rails Autoscale Heroku add-on to automatically scale
   your web dynos.