rails_autoscale_agent 0.9.0.beta.4 → 0.9.0.beta.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70253908c027308ec4d19c9b41eeed033759e6669eace3301925a65a64748447
-  data.tar.gz: 199e75a27f4708920b47964cbc75e2b705ddce88508c35fd614512bb9bfcfd5e
+  metadata.gz: f151c5012b2d2a5bb76f17ed4b4ee7f8f751ef9657c7f4b9d42d4d86fda7c654
+  data.tar.gz: 4773c138b97d723aa44f803ccb504facdf4c619995122296561933b5363d8910
 SHA512:
-  metadata.gz: af5043f8402b40c15b5efe0be205e0e4c6107b8d05ea7126ccfa445c28679567c7e0f15e2a01eb2b7e107c95d3802543c48f428c31f70be823953173cd903ae1
-  data.tar.gz: 1fcfa15d0c76a8f1abf38653890d6b2157fd2e4edfc2ac072eb198897ab09029ea3f1c83ee11620e13a1a07a379695d7913c5bed89ce51e1907451367e61b1df
+  metadata.gz: 432f9ea93f5c722f286a0bec54f22c477583f0b0501cff64498f2d710fdd5e3f674753443d017ab43cefc04d0b8b2263227554992e84f023b13074392d004e15
+  data.tar.gz: 1ba59eb800d04044e78e2c5462fe373dc4d81e1b14de72d4a41f3454acf4a4a0739067f8371ad2d31372badf2c222dffc0b13c2ef0c01e542d12d218a6c1981f

data/.vscode/tasks.json

@@ -6,7 +6,8 @@
     {
       "label": "test: all",
       "group": "test",
-      "command": "rspec",
+      "command": "bundle",
+      "args": ["exec", "rspec"],
       "runOptions": {
         "reevaluateOnRerun": false
       },
@@ -31,8 +32,8 @@
     {
       "label": "test: file",
       "group": "test",
-      "command": "rspec",
-      "args": ["${relativeFile}"],
+      "command": "bundle",
+      "args": ["exec", "rspec", "${relativeFile}"],
       "runOptions": {
         "reevaluateOnRerun": false
       },
@@ -57,8 +58,8 @@
     {
       "label": "test: line",
       "group": "test",
-      "command": "rspec",
-      "args": ["${relativeFile}:${lineNumber}"],
+      "command": "bundle",
+      "args": ["exec", "rspec", "${relativeFile}:${lineNumber}"],
       "runOptions": {
         "reevaluateOnRerun": false
       },

data/Gemfile

@@ -1,4 +1,16 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in rails_autoscale_agent.gemspec
 gemspec
+
+gem "rake", ">= 12.3.3"
+gem "rspec", ">= 3.0"
+gem "vcr", ">= 3.0"
+gem "webmock"
+gem "pry-byebug"
+gem "sidekiq", ">= 5.0"
+gem "delayed_job"
+gem "delayed_job_active_record"
+gem "que"
+gem "resque"
+gem "activesupport"
+gem "sqlite3", platforms: :ruby

data/lib/rails_autoscale_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsAutoscaleAgent
-  VERSION = "0.9.0.beta.4"
+  VERSION = "0.9.0.beta.5"
 end

data/lib/rails_autoscale_agent/worker_adapters/delayed_job.rb

@@ -8,63 +8,50 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
-      UNNAMED_QUEUE = '[unnamed]'
+      attr_writer :queues
 
-      class << self
-        attr_accessor :queues
-      end
-
-      def initialize
+      def queues
         # Track the known queues so we can continue reporting on queues that don't
-        # currently have enqueued jobs.
-        self.class.queues = Set.new
-
-        install if enabled?
+        # have enqueued jobs at the time of reporting.
+        # Assume a "default" queue so we always report *something*, even when nothing
+        # is enqueued.
+        @queues ||= Set.new(['default'])
       end
 
       def enabled?
-        defined? ::Delayed
+        if defined?(::Delayed::Job) && defined?(::Delayed::Backend::ActiveRecord)
+          logger.info "DelayedJob enabled (#{::ActiveRecord::Base.default_timezone})"
+          true
+        end
       end
 
       def collect!(store)
         log_msg = String.new
         t = Time.now.utc
-        db_time = t.strftime('%Y-%m-%d %H:%M:%S')
+        sql = <<~SQL
+          SELECT COALESCE(queue, 'default'), min(run_at)
+          FROM delayed_jobs
+          WHERE locked_at IS NULL
+          AND failed_at IS NULL
+          GROUP BY queue
+        SQL
 
-        # Ignore failed jobs (they skew latency measurement due to the original run_at)
-        sql = "SELECT queue, min(run_at) FROM delayed_jobs WHERE run_at < '#{db_time}' AND locked_at IS NULL AND failed_at IS NULL GROUP BY queue"
         run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-        queues = self.class.queues | run_at_by_queue.keys
+        self.queues |= run_at_by_queue.keys
 
         queues.each do |queue|
           run_at = run_at_by_queue[queue]
-          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          # DateTime.parse assumes a UTC string
+          run_at = DateTime.parse(run_at) if run_at.is_a?(String)
           latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          latency_ms = 0 if latency_ms < 0
 
-          queue = UNNAMED_QUEUE if queue.nil? || queue.empty?
           store.push latency_ms, t, queue
           log_msg << "dj.#{queue}=#{latency_ms} "
         end
 
         logger.debug log_msg unless log_msg.empty?
       end
-
-      private
-
-      def install
-        plugin = Class.new(Delayed::Plugin) do
-          require 'delayed_job'
-
-          callbacks do |lifecycle|
-            lifecycle.before(:enqueue) do |job, &block|
-              queue = job.queue || 'default'
-              WorkerAdapters::DelayedJob.queues.add queue
-            end
-          end
-        end
-
-        Delayed::Worker.plugins << plugin
-      end
     end
   end
 end

data/lib/rails_autoscale_agent/worker_adapters/que.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'rails_autoscale_agent/logger'
-require 'time'
 
 module RailsAutoscaleAgent
   module WorkerAdapters
@@ -9,36 +8,44 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
-      UNNAMED_QUEUE = '[unnamed]'
-      DEFAULT_QUEUES = ['default']
+      attr_writer :queues
 
-      class << self
-        attr_accessor :queues
-      end
-
-      def initialize
-        self.class.queues = DEFAULT_QUEUES
+      def queues
+        # Track the known queues so we can continue reporting on queues that don't
+        # have enqueued jobs at the time of reporting.
+        # Assume a "default" queue so we always report *something*, even when nothing
+        # is enqueued.
+        @queues ||= Set.new(['default'])
       end
 
       def enabled?
-        defined? ::Que
+        if defined?(::Que)
+          logger.info "Que enabled (#{::ActiveRecord::Base.default_timezone})"
+          true
+        end
       end
 
       def collect!(store)
         log_msg = String.new
-        t = Time.now
+        t = Time.now.utc
+        sql = <<~SQL
+          SELECT queue, min(run_at)
+          FROM que_jobs
+          WHERE finished_at IS NULL
+          AND expired_at IS NULL
+          AND error_count = 0
+          GROUP BY 1
+        SQL
 
-        # Ignore failed jobs (they skew latency measurement due to the original run_at)
-        sql = 'SELECT queue, min(run_at) FROM que_jobs WHERE error_count = 0 GROUP BY queue'
         run_at_by_queue = Hash[ActiveRecord::Base.connection.select_rows(sql)]
-        self.class.queues |= run_at_by_queue.keys
+        self.queues |= run_at_by_queue.keys
 
-        self.class.queues.each do |queue|
+        queues.each do |queue|
           run_at = run_at_by_queue[queue]
-          run_at = Time.parse(run_at) if run_at.is_a?(String)
+          run_at = DateTime.parse(run_at) if run_at.is_a?(String)
           latency_ms = run_at ? ((t - run_at)*1000).ceil : 0
+          latency_ms = 0 if latency_ms < 0
 
-          queue = UNNAMED_QUEUE if queue.nil? || queue.empty?
           store.push latency_ms, t, queue
           log_msg << "que.#{queue}=#{latency_ms} "
         end

data/lib/rails_autoscale_agent/worker_adapters/resque.rb

@@ -8,8 +8,15 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
+      attr_writer :queues
+
+      def queues
+        @queues ||= ['default']
+      end
+
       def enabled?
         require 'resque'
+        logger.info "Resque enabled"
         true
       rescue LoadError
         false
@@ -18,7 +25,11 @@ module RailsAutoscaleAgent
       def collect!(store)
         log_msg = String.new
 
-        ::Resque.queues.each do |queue|
+        # Ensure we continue to collect metrics for known queue names, even when nothing is
+        # enqueued at the time. Without this, it will appears that the agent is no longer reporting.
+        self.queues |= ::Resque.queues
+
+        queues.each do |queue|
           next if queue.nil? || queue.empty?
           depth = ::Resque.size(queue)
           store.push depth, Time.now, queue, :qd

data/lib/rails_autoscale_agent/worker_adapters/sidekiq.rb

@@ -8,8 +8,15 @@ module RailsAutoscaleAgent
       include RailsAutoscaleAgent::Logger
       include Singleton
 
+      attr_writer :known_queue_names
+
+      def known_queue_names
+        @known_queue_names ||= ['default']
+      end
+
       def enabled?
         require 'sidekiq/api'
+        logger.info "Sidekiq enabled"
         true
       rescue LoadError
         false
@@ -17,8 +24,18 @@ module RailsAutoscaleAgent
 
       def collect!(store)
         log_msg = String.new
+        queues_by_name = ::Sidekiq::Queue.all.each_with_object({}) do |queue, obj|
+          obj[queue.name] = queue
+        end
+
+        # Ensure we continue to collect metrics for known queue names, even when nothing is
+        # enqueued at the time. Without this, it will appears that the agent is no longer reporting.
+        known_queue_names.each do |queue_name|
+          queues_by_name[queue_name] ||= ::Sidekiq::Queue.new(queue_name)
+        end
+        self.known_queue_names = queues_by_name.keys
 
-        ::Sidekiq::Queue.all.each do |queue|
+        queues_by_name.each do |queue_name, queue|
           latency_ms = (queue.latency * 1000).ceil
           depth = queue.size
           store.push latency_ms, Time.now, queue.name, :qt

data/rails_autoscale_agent.gemspec

@@ -15,16 +15,4 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 12.3.3"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "vcr", "~> 3.0"
-  spec.add_development_dependency "webmock"
-  spec.add_development_dependency "pry"
-  spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "sidekiq", "~> 5.0"
-  spec.add_development_dependency "delayed_job"
-  spec.add_development_dependency "que"
-  spec.add_development_dependency "activesupport"
 end

metadata

@@ -1,169 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: rails_autoscale_agent
 version: !ruby/object:Gem::Version
-  version: 0.9.0.beta.4
+  version: 0.9.0.beta.5
 platform: ruby
 authors:
 - Adam McCrea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-17 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 12.3.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 12.3.3
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: delayed_job
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: que
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+date: 2020-06-28 00:00:00.000000000 Z
+dependencies: []
 description: 
 email:
 - adam@adamlogic.com