sidekiq-heroku-scaler 0.2.0 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a621fafaabccaf23384286cd22585c3438f89cdcdc086f9daae43a8f466b9111
-  data.tar.gz: 6e7332ed2fd08e54ae728837a2ed4555d249b221f2d4bbfcf63d8f510246a2f6
+  metadata.gz: 6992b9c247425e28e7b72e09cbc4b0a78ba8c04fba9084b80dc986acd5bcf261
+  data.tar.gz: f4d7f83e559e4e4cedc2f52415a1d4b5714e4b31f2ea896f6fa78bfb4c6c0f49
 SHA512:
-  metadata.gz: aacc9a300953800982c0dc6ffe14048a8caf9598169182ed0b2ba7c8eb73807eeceb6b4a1adb83537f0f1d881756ed8b10c95d1ca34ab0212a0bf2f0b71ca9d0
-  data.tar.gz: c0941719a3815b45c41d21f26f97c483be59217c98b6d26de8000eca6188a9c86877b743f03e23f26dc3d7d247ed67f8c030ee277a1d09dc7a8bedeaee62b589
+  metadata.gz: 3dc23c537c55cdec634780d1a7195bb2f3c32f23a15d1fad34ddcc8000c82d39a87c1cb81dc4a449474a0e758d2c16429d24280ab64e12849af20cfbf608ad8a
+  data.tar.gz: 7f74a202b4ccbdf3ac295c13e4fb67256cc44735626655241d396eaaf7c1d24334e65a8cc90862d03b91f6bd41e4cb9a800918c838fc4aed3df6de05c2abfc38

data/.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
-  TargetRubyVersion: 2.5
+  NewCops: enable
+  TargetRubyVersion: 2.7
   Exclude:
     - 'bin/*'
 Documentation:

data/Gemfile.lock

@@ -1,67 +1,84 @@
 PATH
   remote: .
   specs:
-    sidekiq-heroku-scaler (0.2.0)
+    sidekiq-heroku-scaler (0.3.3)
+      activesupport (> 5, < 8)
       platform-api (> 3, < 4)
       sidekiq (> 4, < 7)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.0)
-    connection_pool (2.2.5)
-    diff-lcs (1.3)
+    activesupport (7.0.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    ast (2.4.2)
+    concurrent-ruby (1.1.10)
+    connection_pool (2.3.0)
+    diff-lcs (1.5.0)
     erubis (2.7.0)
-    excon (0.92.2)
+    excon (0.92.4)
     heroics (0.1.2)
       erubis (~> 2.0)
       excon
       moneta
       multi_json (>= 1.9.2)
       webrick
-    jaro_winkler (1.5.1)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    json (2.6.2)
+    minitest (5.16.3)
     moneta (1.0.0)
     multi_json (1.15.0)
-    parallel (1.12.1)
-    parser (2.5.3.0)
-      ast (~> 2.4.0)
-    platform-api (3.3.0)
+    parallel (1.22.1)
+    parser (3.1.2.1)
+      ast (~> 2.4.1)
+    platform-api (3.5.0)
       heroics (~> 0.1.1)
       moneta (~> 1.0.0)
       rate_throttle_client (~> 0.1.0)
-    powerpack (0.1.2)
-    rack (2.2.3)
-    rainbow (3.0.0)
-    rake (12.3.2)
+    rack (2.2.4)
+    rainbow (3.1.1)
+    rake (13.0.6)
     rate_throttle_client (0.1.2)
-    redis (4.6.0)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    redis (4.8.0)
+    regexp_parser (2.5.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    rubocop (0.61.1)
-      jaro_winkler (~> 1.5.1)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.1)
+    rubocop (1.36.0)
+      json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 2.5, != 2.5.1.1)
-      powerpack (~> 0.1)
+      parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.20.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.4.0)
-    ruby-progressbar (1.10.0)
-    sidekiq (6.4.1)
-      connection_pool (>= 2.2.2)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.21.0)
+      parser (>= 3.1.1.0)
+    ruby-progressbar (1.11.0)
+    sidekiq (6.5.7)
+      connection_pool (>= 2.2.5)
       rack (~> 2.0)
-      redis (>= 4.2.0)
-    unicode-display_width (1.4.1)
+      redis (>= 4.5.0, < 5)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.3.0)
     webrick (1.7.0)
 
 PLATFORMS

data/lib/sidekiq-heroku-scaler/heroku/client.rb

@@ -17,15 +17,13 @@ module SidekiqHerokuScaler
 
       def formation_for(worker_name)
         SidekiqHerokuScaler::Heroku::Formation.new(
-          formations.detect { |formation| formation['type'] == worker_name } || {}
+          formations.detect { |formation| formation['type'] == worker_name.to_s } || {}
         )
       end
 
       def sidekiq_workers
-        @sidekiq_workers ||= begin
-          formations.select { |formation| formation['command'].match(/sidekiq/) }
-                    .map { |formation| formation['type'] }
-        end
+        @sidekiq_workers ||= formations.select { |formation| formation['command'].match(/sidekiq/) }
+                                       .map { |formation| formation['type'] }
       end
 
       def update_formation(formation_id, quantity)

data/lib/sidekiq-heroku-scaler/sidekiq_config.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'fileutils'
+require 'optparse'
+require 'sidekiq'
+
+module SidekiqHerokuScaler
+  class SidekiqConfig
+    def initialize(commands = [])
+      @commands = commands
+    end
+
+    def config
+      @config ||= ActiveSupport::HashWithIndifferentAccess.new(setup_options(commands))
+    end
+
+    private
+
+    attr_reader :commands
+
+    def load_yaml(src)
+      if Psych::VERSION > '4.0'
+        YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
+      else
+        YAML.safe_load(src)
+      end
+    end
+
+    def option_parser(opts)
+      OptionParser.new do |o|
+        o.on '-c', '--concurrency INT', 'processor threads to use' do |arg|
+          opts[:concurrency] = Integer(arg)
+        end
+
+        o.on '-g', '--tag TAG', 'Process tag for procline' do |arg|
+          opts[:tag] = arg
+        end
+
+        o.on '-q', '--queue QUEUE[,WEIGHT]', 'Queues to process with optional weights' do |arg|
+          queue, weight = arg.split(',')
+          parse_queue opts, queue, weight
+        end
+
+        o.on '-r', '--require [PATH|DIR]', 'Location of Rails application with jobs or file to require' do |arg|
+          opts[:require] = arg
+        end
+
+        o.on '-C', '--config PATH', 'path to YAML config file' do |arg|
+          opts[:config_file] = arg
+        end
+      end
+    end
+
+    def parse_config(path)
+      erb = ERB.new(File.read(path))
+      erb.filename = File.expand_path(path)
+      opts = load_yaml(erb.result) || {}
+
+      if opts.respond_to?(:deep_symbolize_keys!)
+        opts.deep_symbolize_keys!
+      else
+        symbolize_keys_deep!(opts)
+      end
+
+      opts.delete(:strict)
+
+      parse_queues(opts, opts.delete(:queues) || [])
+
+      opts
+    end
+
+    def parse_queues(opts, queues_and_weights)
+      queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
+    end
+
+    def parse_queue(opts, queue, weight = nil)
+      opts[:queues] ||= []
+      opts[:strict] = true if opts[:strict].nil?
+      raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
+
+      opts[:queues] << queue.to_s
+      opts[:strict] = false if weight.to_i.positive?
+    end
+
+    def parse_options(argv)
+      opts = {}
+      parser = option_parser(opts)
+      parser.parse!(argv)
+      opts
+    end
+
+    def setup_options(args)
+      # parse CLI options
+      opts = parse_options(args)
+
+      unless opts[:config_file]
+        config_dir = if File.directory?(opts[:require].to_s)
+                       File.join(opts[:require], 'config')
+                     else
+                       File.join(Sidekiq[:require], 'config')
+                     end
+
+        %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
+          path = File.join(config_dir, config_file)
+          opts[:config_file] ||= path if File.exist?(path)
+        end
+      end
+
+      # parse config file options
+      opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file] && File.exist?(opts[:config_file])
+
+      # set defaults
+      opts[:queues] = ['default'] if opts[:queues].nil?
+      opts[:concurrency] = Integer(ENV['RAILS_MAX_THREADS']) if opts[:concurrency].nil? && ENV['RAILS_MAX_THREADS']
+
+      opts
+    end
+
+    def symbolize_keys_deep!(hash)
+      hash.each_key do |k|
+        symkey = k.respond_to?(:to_sym) ? k.to_sym : k
+        hash[symkey] = hash.delete k
+        symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
+      end
+    end
+  end
+end

data/lib/sidekiq-heroku-scaler/strategy/latency.rb

@@ -23,11 +23,13 @@ module SidekiqHerokuScaler
       end
 
       def decrease?(sidekiq_worker)
-        sidekiq_worker.latency < min_latency && sidekiq_worker.quantity > min_dynos_count
+        sidekiq_worker.latency < min_latency &&
+          sidekiq_worker.quantity > min_dynos_count &&
+          (sidekiq_worker.quantity > 1 || !sidekiq_worker.jobs_running?)
       end
 
       def safe_quantity(quantity)
-        return 1 if quantity <= 0
+        return min_dynos_count if quantity < min_dynos_count
 
         quantity > max_dynos_count ? max_dynos_count : quantity
       end

data/lib/sidekiq-heroku-scaler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqHerokuScaler
-  VERSION = '0.2.0'
+  VERSION = '0.3.3'
 end

data/lib/sidekiq-heroku-scaler/worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
+require 'sidekiq-heroku-scaler/sidekiq_config'
 
 module SidekiqHerokuScaler
   class Worker
@@ -21,6 +22,10 @@ module SidekiqHerokuScaler
       formation.quantity
     end
 
+    def jobs_running?
+      Sidekiq::Workers.new.any? { |_process_id, _thread_id, work| queues.include?(work['queue']) }
+    end
+
     def latency
       queues.sum { |queue| Sidekiq::Queue.new(queue).latency }
     end
@@ -33,6 +38,12 @@ module SidekiqHerokuScaler
 
     attr_reader :formation, :worker_name
 
+    def build_process
+      command = formation.command.gsub(/.*sidekiq(\s|\z)/, '').split
+      sideki_config = SidekiqHerokuScaler::SidekiqConfig.new(command)
+      Sidekiq::Process.new(sideki_config.config)
+    end
+
     def queues
       process['queues'] || []
     end
@@ -42,7 +53,7 @@ module SidekiqHerokuScaler
     end
 
     def process
-      process_set.detect { |p| p.identity.match(/\A#{worker_name}\./) } || {}
+      @process ||= process_set.detect { |p| p.identity.match(/\A#{worker_name}\./) } || build_process
     end
   end
 end

data/lib/sidekiq-heroku-scaler.rb

@@ -3,3 +3,6 @@
 require 'sidekiq-heroku-scaler/strategy/latency'
 require 'sidekiq-heroku-scaler/manager'
 require 'sidekiq-heroku-scaler/version'
+
+module SidekiqHerokuScaler
+end

data/sidekiq-heroku-scaler.gemspec

@@ -27,6 +27,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
 
+  spec.add_dependency 'activesupport', '> 5', '< 8'
   spec.add_dependency 'platform-api', '> 3', '< 4'
   spec.add_dependency 'sidekiq', '> 4', '< 7'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-heroku-scaler
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.3
 platform: ruby
 authors:
 - Vital Ryabchinskiy
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-11 00:00:00.000000000 Z
+date: 2022-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: platform-api
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +148,7 @@ files:
 - lib/sidekiq-heroku-scaler/heroku/client.rb
 - lib/sidekiq-heroku-scaler/heroku/formation.rb
 - lib/sidekiq-heroku-scaler/manager.rb
+- lib/sidekiq-heroku-scaler/sidekiq_config.rb
 - lib/sidekiq-heroku-scaler/strategy/latency.rb
 - lib/sidekiq-heroku-scaler/version.rb
 - lib/sidekiq-heroku-scaler/worker.rb
@@ -135,7 +156,8 @@ files:
 homepage: https://github.com/vitalinfo/sidekiq-heroku-scaler
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -151,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Sidekiq Heroku Scaler