sidekiq-qlimit 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d62aada6eb22eb9c8e3c23a875d538fcabf8613d
+  data.tar.gz: dcac1cc3a9a1e4d98b19ffebf7c717992a2c5863
+SHA512:
+  metadata.gz: dff8269278b27266ac7b846042178d49fd53d089c0022ef623bd2100e5e46c941d926380ef307dcf0e19af5ee258407bb80b3f4dc2a9e045813c62bc58478a44
+  data.tar.gz: 81f1586e34d2697ed9ae2575e70d0891aea34f88d2c3bcd4fd3ad9095a6f4245f88f0bf0dc5b44ab4e26744c7d0e651acbf410c97f74c16bef310848f08889a7

data/lib/sidekiq/qlimit/qlimit_fetch.rb

@@ -0,0 +1,279 @@
+# frozen_string_literal: true
+
+require "celluloid" if Sidekiq::VERSION < "4.0.0"
+require "sidekiq"
+require "sidekiq/fetch"
+
+module Sidekiq
+  module Qlimit
+    ##
+    # Throttled version of `Sidekiq::QlimitFetch` fetcher strategy.
+    #
+    #
+    # Just add somewhere in your bootstrap:
+    #
+    #     require "sidekiq/qlimit"
+    #     Sidekiq::Qlimit.setup!
+    #
+    # Establish max # of total workers per queue 
+    #
+    # sidekiq.yml 
+    # --
+    #
+    #   :qlimit:
+    #     queue_name_1: 2
+    #     queue_name_2: 4
+    #
+    #--
+    # TODO: Store current limits in redis and read from redis to display
+    #++
+    class QlimitFetch < ::Sidekiq::BasicFetch
+    
+      # Redis Script SHA tracking
+      @@qlimit_increment_sha = ""
+      @@qlimit_decrement_sha = ""
+
+      # Qlimit aware UnitOfWork
+      UnitOfWork = Struct.new(:queue, :job) do
+        def acknowledge
+          # Reduce qlimit on acknowledge
+          QlimitFetch.qlimit_decrement(queue_name)
+        end
+
+        def queue_name
+          queue.sub(/.*queue:/, ''.freeze)
+        end
+
+        def requeue
+          Sidekiq.redis do |conn|
+            conn.rpush("queue:#{queue_name}", job)
+          end
+        end
+      end
+
+      ## 
+      # Modified Initialize Function - Reads :qlimit from config source such as sidekiq.yml
+      def initialize(options)
+        super(options)
+        
+        # puts options
+        # {:queues=>["unlimited", "limited"], :labels=>[], :concurrency=>5, :require=>".", :environment=>nil, :timeout=>8, :poll_interval_average=>nil, :average_scheduled_poll_interval=>15, :error_handlers=>[#<Sidekiq::ExceptionHandler::Logger:0x007fda0c30fe38>], :lifecycle_events=>{:startup=>[], :quiet=>[], :shutdown=>[]}, :dead_max_jobs=>10000, :dead_timeout_in_seconds=>15552000, :pidfile=>"tmp/pids/sidekiq.pid", :qlimit=>[{"limited"=>2}, {"fake"=>3}], :config_file=>"config/sidekiq.yml", :strict=>true, :fetch=>Sidekiq::Qlimit::QlimitFetch, :tag=>"example"}
+
+
+        # Get our limits
+        @per_queue_limits = {}
+        unless options[:qlimit].nil?
+          options[:qlimit].each do |limit_hash|
+            limit_hash.each do |k, v|
+              @per_queue_limits[k] = v
+            end
+          end
+        end
+
+        # TODO: Store current limits in redis and read from redis to display
+
+        QlimitFetch.qlimit_script_load
+      end
+
+
+
+
+
+      ## 
+      # Returns an array of queue names that are NOT too busy
+      def qualifying_queues
+        # Working copy of @queues list
+        allowed_queues = @queues.dup
+        
+        # Remove any queue which has hit the maximum number of concurrent jobs 
+        # NOTE: Could remove a queue for which a job has just finished, but we'll catch that job on the next loop
+        @per_queue_limits.each do |k, v|
+              Sidekiq.logger.debug("Checking #{k} => #{v}")
+              Sidekiq.redis do |conn|
+                jobs_in_queue = QlimitFetch.qlimit_get(k)
+                if jobs_in_queue.to_i >= v # detected a maximum concurrent case
+                  allowed_queues.delete("queue:#{k}")
+                  Sidekiq.logger.debug("Remove: queue:#{k}")
+                end
+              end
+                
+        end
+
+        Sidekiq.logger.debug("Allowed Queues: #{allowed_queues}")
+
+        # Follow original sidekiq fetch strategy
+        if @strictly_ordered_queues
+          allowed_queues
+        else
+          allowed_queues = allowed_queues.shuffle.uniq
+          allowed_queues << TIMEOUT # brpop should Wait X number of seconds before returning nil
+          allowed_queues
+        end
+      end
+
+
+      ## 
+      # Returns a "UnitOfWork" from a qualifying queue if available
+      def retrieve_work 
+        work_text = Sidekiq.redis { |conn| conn.brpop(*qualifying_queues) }
+        work = UnitOfWork.new(*work_text) if work_text
+
+        if work.nil?
+          Sidekiq.logger.debug("No Work")
+          return
+        end
+        
+        # We hack around the simultaneous zero starting problem by incrementing an expiring counter 
+        okay_to_continue = QlimitFetch.qlimit_increment(work.queue_name, @per_queue_limits[work.queue_name])
+        Sidekiq.logger.debug("QlimitIncrement Result => #{okay_to_continue}")
+
+        if okay_to_continue
+          Sidekiq.logger.debug("Perform #{work}")
+          return work
+        else
+          Sidekiq.logger.debug("Requeue #{work}")
+          Sidekiq.redis { |conn| conn.lpush(work.queue, work.job) }
+          return nil
+        end
+      end
+
+
+      ## 
+      # Returns a "UnitOfWork" from a qualifying queue if available
+      def self.qlimit_script_load
+        # Note:
+        # This is not theadsafe.  Instead we *blindly* increment if current < max. 
+        # We assume there is little or no penalty for running a few too many workers
+        qlimit_increment_script = <<-EOF
+            local max = tonumber(ARGV[1])
+            local current = tonumber(redis.call('get',KEYS[1]))
+            local current_i = 0
+            if nil == current then
+              current_i = 0
+            else
+              current_i = tonumber(current)
+            end
+
+            if current_i < max then
+                redis.call('incr',KEYS[1])
+                redis.call('expire',KEYS[1], 14400)
+                return true
+            else
+                return false
+            end
+        EOF
+
+        # Note: 
+        # If we zero a counter or reduce a limit, we could go "negative" on a decrement.  
+        # Limit minimum at 0 which is self correcting.
+        qlimit_decrement_script = <<-EOF
+            redis.call('decrby', KEYS[1], ARGV[1])
+            local current = tonumber(redis.call('get',KEYS[1]))
+            if current < 0 then
+                redis.call('set',KEYS[1], 0)
+            end
+            redis.call('expire',KEYS[1], 14400)
+        EOF
+
+        Sidekiq.redis do |conn|
+          @@qlimit_increment_sha = conn.script(:load, qlimit_increment_script)
+          @@qlimit_decrement_sha = conn.script(:load, qlimit_decrement_script)
+        end
+      end
+
+      ## 
+      # Returns a hash of current count of running jobs in queues
+      # 
+      #   Example: { "queue1": 123, "queue2": 456 }
+      def self.qlimit_hash
+        qlimits = {}
+        Sidekiq.redis do |conn|
+          conn.scan_each(match: "qlimit:*") do |key|
+            qkey = key.sub(/.*qlimit:/, ''.freeze)
+            qvalue = conn.get(key)
+            qvalue ||= 0
+            qlimits[qkey] ||= qvalue.to_i
+          end
+        end
+        qlimits
+      end
+
+      ## 
+      # Increment current count of running jobs in queue by amount NOT to exceed limit
+      #
+      # return 1 if incrementing count would NOT exceed limit
+      # return 0 if incrementing count would exceed limit
+      def self.qlimit_increment(queue, limit)
+        return 1 if limit.nil?  # No limit, no processing
+
+        Sidekiq.redis do |conn|
+          result = conn.evalsha(@@qlimit_increment_sha,["qlimit:#{queue}"],[limit])
+          #Sidekiq.logger.debug("Checking Qlimit #{queue} => #{result}")
+          return result
+        end
+      end
+
+      ## 
+      # Decrement current count of running jobs in queue by amount (default: 1)
+      def self.qlimit_decrement(queue, amount = 1)
+        Sidekiq.logger.debug("Qlimit Decrement: #{queue} by #{amount}")
+
+        Sidekiq.redis do |conn|
+          result = conn.evalsha(@@qlimit_decrement_sha,["qlimit:#{queue}"], [amount])
+        end
+      end
+
+      ## 
+      # Set current count of running jobs in queue to 0
+      def self.qlimit_reset(queue)
+          self.qlimit_set(queue, 0)
+      end
+
+      ## 
+      # Set current count of running jobs in queue
+      def self.qlimit_set(queue, amount = 0)
+        Sidekiq.logger.debug("Qlimit Set: #{queue} => #{amount}")
+        Sidekiq.redis do |conn|
+          conn.set("qlimit:#{queue}", amount)
+          conn.expire("qlimit:#{queue}", 14400)
+        end
+      end
+
+      ## 
+      # Get current count of running jobs from queue
+      def self.qlimit_get(queue)
+        Sidekiq.redis do |conn|
+          result = conn.get("qlimit:#{queue}")
+          return result.to_i unless result.nil?
+          return 0
+        end
+      end
+
+      ## 
+      # Used to requeue jobs on sidekiq shutdown/termination
+      def self.bulk_requeue(inprogress, options)
+        return if inprogress.empty?
+
+        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+        jobs_to_requeue = {}
+        inprogress.each do |unit_of_work|
+          jobs_to_requeue[unit_of_work.queue_name] ||= []
+          jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+        end
+
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            jobs_to_requeue.each do |queue, jobs|
+              conn.rpush("queue:#{queue}", jobs)
+              # Reduce qlimit on requeue of amount: jobs.length
+              self.qlimit_decrement(queue, jobs.length)
+            end
+          end
+        end
+        Sidekiq.logger.debug("Pushed #{inprogress.size} jobs back to Redis")
+      rescue => ex
+        Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+      end
+    end
+  end
+end

data/lib/sidekiq/qlimit/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Qlimit
+    # Gem version
+    VERSION = "0.0.4".freeze
+  end
+end

data/lib/sidekiq/qlimit/views/index.html.erb

@@ -0,0 +1,39 @@
+<header class='row'>
+  <div class='span5 col-sm-5'>
+    <h3><%=t 'Qlimits' %></h3>
+  </div>
+</header>
+
+<table class="table table-hover table-bordered table-striped table-white">
+  <thead>
+    <tr>
+      <th>Queue</th>
+      <th style="text-align:center;">Current Running</th>
+      <th style="text-align:center;">Limit</th>
+      <th style="text-align:center;">Zero</th>
+    </tr>
+  </thead>
+  <tbody>
+    <% Sidekiq::Qlimit::QlimitFetch.qlimit_hash.each do |k,v| %>
+      <tr>
+        <td style="vertical-align:middle;text-align:center;">
+          <%= k %>
+        </td>
+        <td style="vertical-align:middle;text-align:center;">
+          <%= v %>
+        </td>
+        <td style="vertical-align:middle;text-align:center;">
+          -Later-
+          <!-- TODO: Store current limits in redis and read from redis to display -->
+        </td>
+        <td style="vertical-align:middle;text-align:center;">
+          <form action="<%= root_path %>qlimit/<%= k %>" method="POST">
+            <%= csrf_tag %>
+            <input type="hidden" name="_method" value="delete" />
+            <button class="btn btn-danger" type="submit">Reset</button>
+          </form>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

data/lib/sidekiq/qlimit/web.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+# stdlib
+require "pathname"
+
+# 3rd party
+require "sidekiq"
+require "sidekiq/web"
+
+# internal
+require "sidekiq/qlimit/web_extension"
+
+
+if defined?(Sidekiq::Web)
+  Sidekiq::Web.register Sidekiq::Qlimit::WebExtension
+
+  if Sidekiq::Web.tabs.is_a?(Array)
+    # For sidekiq < 2.5
+    Sidekiq::Web.tabs << "cron"
+  else
+    Sidekiq::Web.tabs["Qlimit"] = "qlimit"
+  end
+end

data/lib/sidekiq/qlimit/web_extension.rb

@@ -0,0 +1,19 @@
+require 'tilt/erubis'
+
+module Sidekiq
+  module Qlimit
+    module WebExtension
+      def self.registered(app)
+        view_path    = File.join(File.expand_path("..", __FILE__), "views")
+        app.get "/qlimit" do
+            render(:erb, File.read(File.join(view_path, "index.html.erb")))
+        end
+
+        app.delete "/qlimit/:id" do |id|
+          Sidekiq::Qlimit::QlimitFetch.qlimit_reset(id)
+          redirect "#{root_path}qlimit"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/qlimit.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+# 3rd party
+require "sidekiq"
+
+# internal
+require "sidekiq/version"
+require_relative "qlimit/qlimit_fetch"
+require_relative 'qlimit/web'
+
+
+# @see https://github.com/mperham/sidekiq/
+module Sidekiq
+  # Sidekiq per queue 'soft' limiting
+  #
+  # Just add somewhere in your bootstrap (config/initializers/sidekiq-qlimit.rb):
+  #
+  #     require "sidekiq/qlimit"
+  #     Sidekiq::Qlimit.setup!
+  #
+  module Qlimit
+    class << self
+      # Hooks Qlimit into sidekiq.
+      # @return [void]
+      def setup!
+        Sidekiq.configure_server do |config|
+          require "sidekiq/qlimit/qlimit_fetch"
+          Sidekiq.options[:fetch] = Sidekiq::Qlimit::QlimitFetch
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-qlimit
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Jeff Chan
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: tilt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.7
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+description: Sidekiq per queue 'soft' limiting. It ain't perfect, but it's enough.
+email:
+- jeff@braincommerce.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sidekiq/qlimit.rb
+- lib/sidekiq/qlimit/qlimit_fetch.rb
+- lib/sidekiq/qlimit/version.rb
+- lib/sidekiq/qlimit/views/index.html.erb
+- lib/sidekiq/qlimit/web.rb
+- lib/sidekiq/qlimit/web_extension.rb
+homepage: https://github.com/braincom/sidekiq-qlimit
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--title"
+- Sidekiq-Qlimit - A Soft Limiter
+- "--main"
+- README.md
+- "--exclude"
+- example
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Sidekiq per queue 'soft' limiting.
+test_files: []