sidekiq_autoscale 0.2.3 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cfb3c09aae5e59a3b70e5f59dfcec134f6fdd76967d85dde6a938fd923e7798
-  data.tar.gz: 1ab3220369cc25eb91b3b38181c5a1c54cae20898afae858a3d56b275346344e
+  metadata.gz: 35870ccd72ae7e188ecde7da83579cc245dcfaef933796e446c29796bb834a08
+  data.tar.gz: f1c34d154270893707aa7fc0b2888f8c89008733f9db5e6305a5611befd8dff7
 SHA512:
-  metadata.gz: 0e2f36159a9cc83d9f8f9b46b7b902758e966281b2e28d73fcfa8dd8b5559ad97ff58b0bd7eeb36de5cf52868b24c430786ab8c65c6bb71675e4c7ae52e61913
-  data.tar.gz: 8d1ae14cdcb5e82d7a355337f5836ee1c21cb0672d475f3aa54ac0efa81512789886ba35423071cc4bb7c7b5c25a7af493d2fe9b6ac81517a1e6ba448692e833
+  metadata.gz: 3677436e3a7872319dca5d7549c0bf611b428fb8192f4e377e4983751ca2f9ff10196c9d5811efa66e9cba5589b92750d925776ad38f33f17c99cfb88d0f99ef
+  data.tar.gz: 9e059d8e41ee804561d24ac986ce535d228c36541514cd8ee6d57094cbb70dc308f593edcb2346084a04e5b92a036bf5a8f22a0e4f4199f09dbcbe547c245051

data/README.md

@@ -148,5 +148,33 @@ metadata:
 
 Then assign your sidekiq deployment the `myapp-sidekiq` service account.
 
+## Multiple sidekiq pools
+
+Sidekiq autoscaler works with multiple Sidekiq pools that run off a single redis instance.
+
+A typical use case for this would be a client pushing jobs to the same Redis backend,
+which are then processed by multiple sets of workers (i.e. high CPU vs. high memory instances).
+
+If so, you need to set two environment variables in each pool:
+* `SIDEKIQ_QUEUES` - list of queues to be processed
+* `SIDEKIQ_POOL` - name of the pool
+
+You also need to use `$SIDEKIQ_POOL` tag when starting sidekiq processs.
+
+Example:
+
+```
+# High priority pool
+SIDEKIQ_QUEUES='["critical", "urgent"]'
+SIDEKIQ_POOL=high
+
+# Low priority pool
+SIDEKIQ_QUEUES='["default", "low", "searchkick"]'
+SIDEKIQ_POOL=low
+
+# Start sidekiq
+bundle exec sidekiq -C config/sidekiq.yml -g $SIDEKIQ_POOL
+```
+
 ## License
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/sidekiq_autoscale/adapters/kubernetes_adapter.rb

@@ -15,9 +15,12 @@ module SidekiqAutoscale
 
     def worker_count
       @resources.get(@deployment_name).spec.replicas
-    rescue Excon::Errors::Error, K8s::Error, K8s::Error::Forbidden => e
+    rescue K8s::Error, K8s::Error::Forbidden => e
       SidekiqAutoscale.on_scaling_error(e)
       0
+    rescue Excon::Errors::Error, Excon::Error::Socket => e
+      SidekiqAutoscale.logger.warn("[SIDEKIQ_SCALE][KUBERNETES_ACTION] Error connecting to Kubernetes: #{e}")
+      0
     end
 
     def worker_count=(val)

data/lib/sidekiq_autoscale/middleware.rb

@@ -2,10 +2,11 @@
 
 module SidekiqAutoscale
   class Middleware
-    LAST_SCALED_AT_EVENT_KEY = "sidekiq_autoscaling:last_scaled_at"
-    SCALING_LOCK_KEY = "sidekiq_autoscaling:scaling_lock"
+    POOL_NAME = ENV.fetch("SIDEKIQ_POOL", "default")
+    LAST_SCALED_AT_EVENT_KEY = "sidekiq_autoscaling:#{POOL_NAME}:last_scaled_at"
+    SCALING_LOCK_KEY = "sidekiq_autoscaling:#{POOL_NAME}:scaling_lock"
     LOG_TAG = "[SIDEKIQ_SCALE][SCALING_EVENT]"
-    WORKER_COUNT_KEY = "sidekiq_autoscaling/current_worker_count"
+    WORKER_COUNT_KEY = "sidekiq_autoscaling/#{POOL_NAME}/current_worker_count"
 
     # @param [Object] worker the worker instance
     # @param [Hash] job the full job payload

data/lib/sidekiq_autoscale/sidekiq_interface.rb

@@ -9,7 +9,12 @@ module SidekiqAutoscale
     end
 
     def queue_names
-      ::Sidekiq::Queue.all.map(&:name)
+      from_env = ENV["SIDEKIQ_QUEUES"]
+      if from_env
+        JSON.parse(from_env)
+      else
+        ::Sidekiq::Queue.all.map(&:name)
+      end
     end
 
     def busy_threads
@@ -39,7 +44,13 @@ module SidekiqAutoscale
     private
 
     def process_set
-      @process_set ||= ::Sidekiq::ProcessSet.new
+      @process_set ||= begin
+        ps = ::Sidekiq::ProcessSet.new
+        pool = ENV["SIDEKIQ_POOL"]
+
+        ps = ps.select {|process| process.tag == pool } if pool
+        ps
+      end
     end
   end
 end

data/lib/sidekiq_autoscale/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqAutoscale
-  VERSION = "0.2.3"
+  VERSION = "0.2.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq_autoscale
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.6
 platform: ruby
 authors:
 - Steven Allen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-09 00:00:00.000000000 Z
+date: 2022-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -56,14 +56,14 @@ dependencies:
   name: railties
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.6
 - !ruby/object:Gem::Dependency
@@ -358,7 +358,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.32
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: A simple gem to handle Sidekiq autoscaling.