sidekiq-limit_fetch 2.4.2 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1edc411f8e5b3ab852b0b005548a28fafd0167e4
-  data.tar.gz: df6b938724aa978eab9757d1c969c9577c9e58c3
+  metadata.gz: 76501c38471c02fe380102202fbd2afb9799976e
+  data.tar.gz: 20d1ed4d5d36ca62b7dcff40e09842378ba1d3a1
 SHA512:
-  metadata.gz: 4cb020c997e1dc814969e949ae497f49ee048f4d8b0b18833dd8eca51de0a1b943cae09d2e34e523e9d9c29b251101bf707cbab67d7a5ff270cda43be69ffbbd
-  data.tar.gz: 29eb71b1e70bde0da6dee13798e2fae39c2900abca6cbcf25d22ab2466028525ef84ff4abb3c00b02b48524b4ffef09bb8a98f432761d9e5081bf3a656c7d24e
+  metadata.gz: b14c94fad7ef434afc4b16e58f3517a426bf1f95f31aac2b12b279e4f72e70ebb11746b58ca1873d582a32568c5d27cc6fe05daf28f5a3e0f299361f7ebdc015
+  data.tar.gz: 030fe7be744ff109d6ff17740e71576f9011f432255ce00ab5423dcd5486e75d2a76cf9e0c06825f5fbc962aab6fbe6a4d3a4e46931698006e3e9e39467ae3c7

data/.gitignore

@@ -1,2 +1,3 @@
 Gemfile.lock
 pkg/
+.bundle/

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper --color

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 gemspec
 
-gem 'sidekiq', github: 'mperham/sidekiq'
+gem 'sidekiq', github: 'mperham/sidekiq', branch: 'internal_rewrite'

data/README.md

@@ -18,7 +18,10 @@ Add this line to your application's Gemfile:
 
 redis 2.6 or newer
 
-**Important note:** At this moment, `sidekiq-limit_fetch` is incompatible with Sidekiq Pro's `reliable_fetch`.
+**Important note:** At this moment, `sidekiq-limit_fetch` is incompatible with
+- sidekiq pro's `reliable_fetch`
+- `sidekiq-rate-limiter`
+- any other plugin that rewrites fetch strategy of sidekiq.
 
 https://github.com/mperham/sidekiq/issues/1508 — a sidekiq admin busy page is unreliable, so if you need to check how many workers are running a queue use a `#busy` method described below.
 
@@ -146,6 +149,9 @@ To use this mode you need to specify a following line in sidekiq.yml:
 
 Dynamic queues will be ran at the lowest priority.
 
+### Maintenance
+
+If you use ```flushdb```, restart the sidekiq process to re-populate the dynamic configuration.
 
 ### Thanks
 

data/lib/sidekiq/extensions/manager.rb

@@ -0,0 +1,16 @@
+class Sidekiq::Manager
+  module InitLimitFetch
+    def initialize(options={})
+      options[:fetch] = Sidekiq::LimitFetch
+      super
+    end
+
+    def start
+      Sidekiq::LimitFetch::Queues.start options
+      Global::Monitor.start!
+      super
+    end
+  end
+
+  prepend InitLimitFetch
+end

data/lib/sidekiq/extensions/queue.rb

@@ -1,6 +1,6 @@
 module Sidekiq
   class Queue
-    extend LimitFetch::Singleton, Forwardable
+    extend LimitFetch::Instances, Forwardable
     attr_reader :rname
 
     def_delegators :lock,

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -1,6 +1,5 @@
 module Sidekiq::LimitFetch::Global
   module Monitor
-    include Sidekiq::LimitFetch::Redis
     extend self
 
     HEARTBEAT_PREFIX = 'limit:heartbeat:'
@@ -8,10 +7,10 @@ module Sidekiq::LimitFetch::Global
     HEARTBEAT_TTL = 20
     REFRESH_TIMEOUT = 5
 
-    def start!(queues, ttl=HEARTBEAT_TTL, timeout=REFRESH_TIMEOUT)
+    def start!(ttl=HEARTBEAT_TTL, timeout=REFRESH_TIMEOUT)
       Thread.new do
         loop do
-          add_dynamic queues if queues.dynamic?
+          add_dynamic_queues
           update_heartbeat ttl
           invalidate_old_processes
           sleep timeout
@@ -20,23 +19,24 @@ module Sidekiq::LimitFetch::Global
     end
 
     def all_processes
-      redis {|it| it.smembers PROCESS_SET }
+      Sidekiq.redis {|it| it.smembers PROCESS_SET }
     end
 
     def old_processes
       all_processes.reject do |process|
-        redis {|it| it.get heartbeat_key process }
+        Sidekiq.redis {|it| it.get heartbeat_key process }
       end
     end
 
     def remove_old_processes!
-      redis do |it|
+      Sidekiq.redis do |it|
         old_processes.each {|process| it.srem PROCESS_SET, process }
       end
     end
 
-    def add_dynamic(queues)
-      queues.add Sidekiq::Queue.all.map(&:name)
+    def add_dynamic_queues
+      queues = Sidekiq::LimitFetch::Queues
+      queues.add Sidekiq::Queue.all.map(&:name) if queues.dynamic?
     end
 
     private

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -1,7 +1,5 @@
 module Sidekiq::LimitFetch::Global
   class Semaphore
-    include Sidekiq::LimitFetch::Redis
-
     PREFIX = 'limit_fetch'
 
     attr_reader :local_busy
@@ -45,7 +43,7 @@ module Sidekiq::LimitFetch::Global
     end
 
     def acquire
-      Selector.acquire([@name], determine_namespace).size > 0
+      Selector.acquire([@name], namespace).size > 0
     end
 
     def release
@@ -163,5 +161,15 @@ module Sidekiq::LimitFetch::Global
         it.lrem "#{PREFIX}:busy:#@name", 0, process
       end
     end
+
+    private
+
+    def redis(&block)
+      Sidekiq.redis(&block)
+    end
+
+    def namespace
+      Sidekiq::LimitFetch::Queues.namespace
+    end
   end
 end

data/lib/sidekiq/limit_fetch/{singleton.rb → instances.rb}

@@ -1,4 +1,4 @@
-module Sidekiq::LimitFetch::Singleton
+module Sidekiq::LimitFetch::Instances
   def self.extended(klass)
     klass.instance_variable_set :@instances, {}
   end

data/lib/sidekiq/limit_fetch/queues.rb

@@ -1,88 +1,97 @@
-class Sidekiq::LimitFetch
-  class Queues
-    THREAD_KEY = :acquired_queues
+module Sidekiq::LimitFetch::Queues
+  extend self
 
-    def initialize(options)
-      @queues    = options[:queues]
-      @namespace = options[:namespace]
-      @dynamic   = options[:dynamic]
+  THREAD_KEY = :acquired_queues
 
-      options[:strict] ? strict_order! : weighted_order!
+  def start(options)
+    @queues    = options[:queues]
+    @dynamic   = options[:dynamic]
 
-      set :process_limit, options[:process_limits]
-      set :limit, options[:limits]
-      set_blocks options[:blocking]
-    end
+    options[:strict] ? strict_order! : weighted_order!
 
-    def acquire
-      selector.acquire(ordered_queues, @namespace)
-        .tap {|it| save it }
-        .map {|it| "queue:#{it}" }
-    end
+    set :process_limit, options[:process_limits]
+    set :limit, options[:limits]
+    set_blocks options[:blocking]
+  end
 
-    def release_except(full_name)
-      queues = restore
-      queues.delete full_name[/queue:(.*)/, 1] if full_name
-      selector.release queues, @namespace
-    end
+  def acquire
+    selector.acquire(ordered_queues, namespace)
+      .tap {|it| save it }
+      .map {|it| "queue:#{it}" }
+  end
 
-    def dynamic?
-      @dynamic
-    end
+  def release_except(full_name)
+    queues = restore
+    queues.delete full_name[/queue:(.*)/, 1] if full_name
+    selector.release queues, namespace
+  end
 
-    def add(queues)
-      queues.each do |queue|
-        @queues.push queue unless @queues.include? queue
-      end
-    end
+  def dynamic?
+    @dynamic
+  end
 
-    def strict_order!
-      @queues.uniq!
-      def ordered_queues; @queues end
+  def add(queues)
+    queues.each do |queue|
+      @queues.push queue unless @queues.include? queue
     end
+  end
 
-    def weighted_order!
-      def ordered_queues; @queues.shuffle.uniq end
-    end
+  def strict_order!
+    @queues.uniq!
+    def ordered_queues; @queues end
+  end
 
-    private
+  def weighted_order!
+    def ordered_queues; @queues.shuffle.uniq end
+  end
 
-    def selector
-      Global::Selector
+  def namespace
+    @namespace ||= Sidekiq.redis do |it|
+      if it.respond_to?(:namespace) and it.namespace
+        it.namespace + ':'
+      else
+        ''
+      end
     end
+  end
 
-    def set(limit_type, limits)
-      limits ||= {}
-      each_queue do |queue|
-        limit = limits[queue.name.to_s] || limits[queue.name.to_sym]
-        queue.send "#{limit_type}=", limit unless queue.limit_changed?
-      end
+  private
+
+  def selector
+    Sidekiq::LimitFetch::Global::Selector
+  end
+
+  def set(limit_type, limits)
+    limits ||= {}
+    each_queue do |queue|
+      limit = limits[queue.name.to_s] || limits[queue.name.to_sym]
+      queue.send "#{limit_type}=", limit unless queue.limit_changed?
     end
+  end
 
-    def set_blocks(blocks)
-      each_queue(&:unblock)
+  def set_blocks(blocks)
+    each_queue(&:unblock)
 
-      blocks.to_a.each do |it|
-        if it.is_a? Array
-          it.each {|name| Sidekiq::Queue[name].block_except it }
-        else
-          Sidekiq::Queue[it].block
-        end
+    blocks.to_a.each do |it|
+      if it.is_a? Array
+        it.each {|name| Sidekiq::Queue[name].block_except it }
+      else
+        Sidekiq::Queue[it].block
       end
     end
+  end
 
-    def save(queues)
-      Thread.current[THREAD_KEY] = queues
-    end
+  def save(queues)
+    Thread.current[THREAD_KEY] = queues
+  end
 
-    def restore
-      Thread.current[THREAD_KEY] || []
-    ensure
-      Thread.current[THREAD_KEY] = nil
-    end
+  def restore
+    Thread.current[THREAD_KEY] || []
+  ensure
+    Thread.current[THREAD_KEY] = nil
+  end
 
-    def each_queue
-      @queues.uniq.each {|it| yield Sidekiq::Queue[it] }
-    end
+  def each_queue
+    @queues.uniq.each {|it| yield Sidekiq::Queue[it] }
   end
 end

data/lib/sidekiq/limit_fetch.rb

@@ -1,56 +1,39 @@
+require 'forwardable'
 require 'sidekiq'
-require 'sidekiq/util'
+require 'sidekiq/manager'
 require 'sidekiq/api'
-require 'forwardable'
 
-class Sidekiq::LimitFetch
+module Sidekiq::LimitFetch
   autoload :UnitOfWork, 'sidekiq/limit_fetch/unit_of_work'
 
-  require_relative 'limit_fetch/redis'
-  require_relative 'limit_fetch/singleton'
+  require_relative 'limit_fetch/instances'
   require_relative 'limit_fetch/queues'
   require_relative 'limit_fetch/global/semaphore'
   require_relative 'limit_fetch/global/selector'
   require_relative 'limit_fetch/global/monitor'
   require_relative 'extensions/queue'
+  require_relative 'extensions/manager'
 
-  include Redis
-  Sidekiq.options[:fetch] = self
-
-  def self.bulk_requeue(*args)
-    Sidekiq::BasicFetch.bulk_requeue *args
-  end
+  extend self
 
-  def initialize(options)
-    @queues = Queues.new options.merge(namespace: determine_namespace)
-    Global::Monitor.start! @queues
+  def new(_)
+    self
   end
 
   def retrieve_work
-    queue, message = fetch_message
+    queue, message = redis_brpop *Queues.acquire, Sidekiq::BasicFetch::TIMEOUT
+    Queues.release_except queue
     UnitOfWork.new queue, message if message
   end
 
-  private
-
-  def fetch_message
-    queue, _ = redis_brpop *@queues.acquire, Sidekiq::Fetcher::TIMEOUT
-  ensure
-    @queues.release_except queue
+  def bulk_requeue(*args)
+    Sidekiq::BasicFetch.bulk_requeue(*args)
   end
 
+  private
+
   def redis_brpop(*args)
     return if args.size < 2
-    query = -> redis { redis.brpop *args }
-
-    if busy_local_queues.any? {|queue| not args.include? queue.rname }
-      nonblocking_redis(&query)
-    else
-      redis(&query)
-    end
-  end
-
-  def busy_local_queues
-    Sidekiq::Queue.instances.select(&:local_busy?)
+    Sidekiq.redis {|it| it.brpop *args }
   end
 end

data/sidekiq-limit_fetch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'sidekiq-limit_fetch'
-  gem.version       = '2.4.2'
+  gem.version       = '3.0.0'
   gem.license       = 'MIT'
   gem.authors       = 'brainopia'
   gem.email         = 'brainopia@evilmartians.com'
@@ -13,9 +13,9 @@ Gem::Specification.new do |gem|
 
   gem.files         = `git ls-files`.split($/)
   gem.test_files    = gem.files.grep %r{^spec/}
-  gem.require_paths = %w(lib)
+  gem.require_paths = 'lib'
 
-  gem.add_dependency 'sidekiq', '>= 2.6.5', '< 4.0'
-  gem.add_development_dependency 'rspec', '~> 3.2.0'
+  gem.add_dependency 'sidekiq', '>= 4'
+  gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'rake'
 end

data/spec/sidekiq/extensions/queue_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 RSpec.describe Sidekiq::Queue do
   context 'singleton' do
     shared_examples :constructor do

data/spec/sidekiq/limit_fetch/global/monitor_spec.rb

@@ -1,21 +1,11 @@
-require 'spec_helper'
-
-Thread.abort_on_exception = true
-
 RSpec.describe Sidekiq::LimitFetch::Global::Monitor do
-  let(:queues) { double dynamic?: false }
-  let(:monitor) { described_class.start! queues, ttl, timeout }
+  let(:monitor) { described_class.start! ttl, timeout }
   let(:ttl) { 1 }
   let(:queue) { Sidekiq::Queue[name] }
   let(:name) { 'default' }
 
-  before :each do
-    monitor
-  end
-
-  after :each do
-    monitor.kill
-  end
+  before { monitor }
+  after { monitor.kill }
 
   context 'old locks' do
     let(:timeout) { 0.5 }

data/spec/sidekiq/limit_fetch/queues_spec.rb

@@ -1,8 +1,4 @@
-require 'spec_helper'
-
 RSpec.describe Sidekiq::LimitFetch::Queues do
-  subject { described_class.new options }
-
   let(:queues)         { %w[queue1 queue2] }
   let(:limits)         {{ 'queue1' => 3 }}
   let(:strict)         { true }
@@ -14,10 +10,11 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
       limits:   limits,
       strict:   strict,
       blocking: blocking,
-      process_limits: process_limits,
-      namespace: Sidekiq::LimitFetch::Redis.determine_namespace }
+      process_limits: process_limits }
   end
 
+  before { subject.start options }
+
   it 'should acquire queues' do
     subject.acquire
     expect(Sidekiq::Queue['queue1'].probed).to eq 1

data/spec/sidekiq/limit_fetch/semaphore_spec.rb

@@ -1,5 +1,3 @@
-require 'spec_helper'
-
 RSpec.describe 'semaphore' do
   let(:name) { 'default' }
   subject { Sidekiq::LimitFetch::Global::Semaphore.new name }

data/spec/sidekiq/limit_fetch_spec.rb

@@ -1,7 +1,7 @@
-require 'spec_helper'
+Thread.abort_on_exception = true
 
 RSpec.describe Sidekiq::LimitFetch do
-  before :each do
+  before do
     Sidekiq.redis do |it|
       it.del 'queue:queue1'
       it.lpush 'queue:queue1', 'task1'
@@ -10,11 +10,12 @@ RSpec.describe Sidekiq::LimitFetch do
     end
   end
 
-  subject { described_class.new options }
   let(:options) {{ queues: queues, limits: limits }}
   let(:queues) { %w(queue1 queue1 queue2 queue2) }
   let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
 
+  before { subject::Queues.start options }
+
   it 'should acquire lock on queue for execution' do
     work = subject.retrieve_work
     expect(work.queue_name).to eq 'queue1'

data/spec/spec_helper.rb

@@ -1,6 +1,4 @@
 require 'sidekiq/limit_fetch'
-require 'celluloid/autostart'
-require 'sidekiq/fetch'
 
 Sidekiq.logger = nil
 Sidekiq.redis = { namespace: ENV['namespace'] }
@@ -9,7 +7,7 @@ RSpec.configure do |config|
   config.order = :random
   config.disable_monkey_patching!
   config.raise_errors_for_deprecations!
-  config.before :each do
+  config.before do
     Sidekiq::Queue.reset_instances!
     Sidekiq.redis do |it|
       clean_redis = ->(queue) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-limit_fetch
 version: !ruby/object:Gem::Version
-  version: 2.4.2
+  version: 3.0.0
 platform: ruby
 authors:
 - brainopia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-09 00:00:00.000000000 Z
+date: 2015-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -16,34 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.5
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.5
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -67,6 +61,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - ".travis.yml"
 - Gemfile
 - LICENSE.txt
@@ -86,14 +81,14 @@ files:
 - demo/config/environment.rb
 - demo/config/environments/development.rb
 - lib/sidekiq-limit_fetch.rb
+- lib/sidekiq/extensions/manager.rb
 - lib/sidekiq/extensions/queue.rb
 - lib/sidekiq/limit_fetch.rb
 - lib/sidekiq/limit_fetch/global/monitor.rb
 - lib/sidekiq/limit_fetch/global/selector.rb
 - lib/sidekiq/limit_fetch/global/semaphore.rb
+- lib/sidekiq/limit_fetch/instances.rb
 - lib/sidekiq/limit_fetch/queues.rb
-- lib/sidekiq/limit_fetch/redis.rb
-- lib/sidekiq/limit_fetch/singleton.rb
 - lib/sidekiq/limit_fetch/unit_of_work.rb
 - sidekiq-limit_fetch.gemspec
 - spec/sidekiq/extensions/queue_spec.rb

data/lib/sidekiq/limit_fetch/redis.rb

@@ -1,35 +0,0 @@
-module Sidekiq::LimitFetch::Redis
-  extend self
-
-  def nonblocking_redis
-    redis do |redis|
-      # Celluloid 0.16 broke this method
-      if Celluloid::VERSION.to_f >= 0.16
-        yield redis
-      else
-      # prevent blocking of fetcher
-      # more bullet-proof and faster (O_O)
-      # than using Celluloid::IO
-      #
-      # https://github.com/brainopia/sidekiq-limit_fetch/issues/41
-      # explanation of why Future#value is beneficial here
-        begin
-          Celluloid::Future.new { yield redis }.value
-        rescue Celluloid::Task::TerminatedError
-        end
-      end
-    end
-  end
-
-  def redis
-    Sidekiq.redis {|it| yield it }
-  end
-
-  def determine_namespace
-    redis do |it|
-      if it.respond_to?(:namespace) and it.namespace
-        it.namespace + ':'
-      end
-    end
-  end
-end