sidekiq-prioritized_queues 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 472c6683a50393966af89094f5c98d32d285f7c8
+  data.tar.gz: 86a3a16e91c457682e6e7d95980ed29fac348dbf
+SHA512:
+  metadata.gz: 0b2dee492c651c8c3db1ad0e73f0904cb4ee1b4f337c50241ab3f593fd73f560fb48b20e77eb6253314bc3d1ddaf28b7cc84728eae48227b7aac419f1b6d7ea1
+  data.tar.gz: 2bd344b5c99597085746c3e66563c4c4bc7f458d48ec15c9958d519bf6dbf80952896b9656f77d7120c90a4bbc2b7984e931fd938e3c014430da7b1ae02ae610

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.1.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-prioritized_queues.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Publitas
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+# Sidekiq::PrioritizedQueues
+
+Adds numeric based priorities to your jobs. This is done by monkey patching the following classes:
+
+- `Sidekiq::Client`
+- `Sidekiq::Stats`
+- `Sidekiq::Queue`
+- `Sidekiq::Job`
+
+This gem also adds a new priority based Fetcher, and a middleware that sets priority on the jobs.
+
+**WARNING: This changes the type of the `queue:<name>` keys. There's no migration helper in place, so the easiest way is to start off with a clean setup.**
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-prioritized_queues'
+```
+
+## Usage
+
+Simply having the gem in your Gemfile is enough to get started with prioritized jobs. As a default, priority will be the timestamp at which the job is enqueued, as to simulate the default FIFO order.
+
+There are two ways of specifying priority:
+
+### Static
+
+This is useful whenever different workers share the same queue. An example would be:
+
+```ruby
+class SatisfySameDayOrderWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: 'orders', priority: 0
+
+  def perform(order)
+    # Do stuff here
+  end
+end
+
+class SatisfyOrderWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: 'orders', priority: 1
+
+  def perform(order)
+    # Do stuff here
+  end
+end
+```
+
+For workers on the same queue, the `priority` option can take a `Proc` which will be given the arguments the job was queued with. As an example:
+
+```ruby
+class HeavyWorker
+  include Sidekiq::Worker
+  sidekiq_options priority: -> (account_id) {
+    Account.find(account_id).vip? ? 0 : 10
+  }
+
+  def perform(account_id)
+    # Do some work.
+  end
+end
+```
+
+The example above would make sure that VIP accounts get processed first.
+
+## Contributing
+
+1. Fork it ( https://github.com/publitas/sidekiq-prioritized_queues/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test
+

data/lib/sidekiq/prioritized_queues.rb

@@ -0,0 +1,21 @@
+require 'sidekiq/prioritized_queues/version'
+require 'sidekiq/prioritized_queues/middleware'
+require 'sidekiq/prioritized_queues/fetch'
+require 'sidekiq/prioritized_queues/monkeypatches'
+
+# Add the Client middleware that takes care of setting up the priority property
+# on the messages being queued.
+Sidekiq.configure_server do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::PrioritizedQueues::Middleware
+  end
+end
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::PrioritizedQueues::Middleware
+  end
+end
+
+# Set up the fetcher as the priority based one too.
+Sidekiq.options[:fetch] = Sidekiq::PrioritizedQueues::Fetch

data/lib/sidekiq/prioritized_queues/fetch.rb

@@ -0,0 +1,73 @@
+module Sidekiq
+  module PrioritizedQueues
+    class Fetch
+      def initialize(options)
+        @strictly_ordered_queues = !!options[:strict]
+        @queues = options[:queues].map { |q| "queue:#{q}" }
+        @unique_queues = @queues.uniq
+      end
+
+      def retrieve_work
+        work = nil
+
+        Sidekiq.redis do |conn|
+          queues.find do |queue|
+            response = conn.multi do
+              conn.zrange(queue, 0, 0)
+              conn.zremrangebyrank(queue, 0, 0)
+            end.flatten(1)
+
+            next if response.length == 1
+            work = [queue, response.first]
+            break
+          end
+        end
+
+        return UnitOfWork.new(*work) if work
+        sleep 1; nil
+      end
+
+      # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
+      # an instance method will make it async to the Fetcher actor
+      def self.bulk_requeue(inprogress, options)
+        return if inprogress.empty?
+
+        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+        jobs_to_requeue = {}
+        inprogress.each do |unit_of_work|
+          jobs_to_requeue[unit_of_work.queue_name] ||= []
+          jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.message
+        end
+
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            jobs_to_requeue.each do |queue, jobs|
+              jobs.each { |job| conn.zadd("queue:#{queue}", 0, job) }
+            end
+          end
+        end
+        Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
+      rescue => ex
+        Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+      end
+
+      UnitOfWork = Struct.new(:queue, :message) do
+        def acknowledge
+          # nothing to do
+        end
+
+        def queue_name
+          queue.gsub(/.*queue:/, '')
+        end
+
+        def requeue
+          Sidekiq.redis { |conn| conn.zadd("queue:#{queue_name}", 0, message) }
+        end
+      end
+
+      def queues
+        @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq
+      end
+    end
+  end
+end

data/lib/sidekiq/prioritized_queues/middleware.rb

@@ -0,0 +1,22 @@
+module Sidekiq
+  module PrioritizedQueues
+    class Middleware
+      def call(worker_class, msg, queue, redis_pool)
+        klass = case worker_class
+        when String then worker_class.constantize
+        else worker_class
+        end
+
+        priority = klass.get_sidekiq_options['priority']
+
+        msg['priority'] = case priority
+        when Proc   then priority.call(*msg['args'])
+        when String then priority.to_i
+        else Time.now.to_f
+        end
+
+        yield
+      end
+    end
+  end
+end

data/lib/sidekiq/prioritized_queues/monkeypatches.rb

@@ -0,0 +1,3 @@
+require 'sidekiq/prioritized_queues/monkeypatches/api'
+require 'sidekiq/prioritized_queues/monkeypatches/client'
+require 'sidekiq/prioritized_queues/monkeypatches/web'

data/lib/sidekiq/prioritized_queues/monkeypatches/api.rb

@@ -0,0 +1,70 @@
+require 'sidekiq/api'
+
+module Sidekiq
+  class Stats
+    def queues
+      Sidekiq.redis do |conn|
+        queues = conn.smembers('queues')
+
+        lengths = conn.pipelined do
+          queues.each do |queue|
+            conn.zcard("queue:#{queue}")
+          end
+        end
+
+        i = 0
+        array_of_arrays = queues.inject({}) do |memo, queue|
+          memo[queue] = lengths[i]
+          i += 1
+          memo
+        end.sort_by { |_, size| size }
+
+        Hash[array_of_arrays.reverse]
+      end
+    end
+  end
+
+  class Queue
+    def size
+      Sidekiq.redis { |conn| conn.zcard(@rname) }
+    end
+
+    def latency
+      entry = Sidekiq.redis do |conn|
+        conn.zrange(@rname, -1, -1)
+      end.first
+      return 0 unless entry
+      Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at']
+    end
+
+    def each(&block)
+      initial_size = size
+      deleted_size = 0
+      page = 0
+      page_size = 50
+
+      loop do
+        range_start = page * page_size - deleted_size
+        range_end   = page * page_size - deleted_size + (page_size - 1)
+        entries = Sidekiq.redis do |conn|
+          conn.zrevrange @rname, range_start, range_end
+        end
+        break if entries.empty?
+        page += 1
+        entries.each do |entry|
+          block.call Job.new(entry, @name)
+        end
+        deleted_size = initial_size - size
+      end
+    end
+  end
+
+  class Job
+    def delete
+      count = Sidekiq.redis do |conn|
+        conn.zrem("queue:#{@queue}", @value)
+      end
+      count != 0
+    end
+  end
+end

data/lib/sidekiq/prioritized_queues/monkeypatches/client.rb

@@ -0,0 +1,25 @@
+module Sidekiq
+  class Client
+
+  private
+
+    def atomic_push(conn, payloads)
+      if payloads.first['at']
+        conn.zadd('schedule', payloads.map do |hash|
+          at = hash.delete('at').to_s
+          [at, Sidekiq.dump_json(hash)]
+        end)
+      else
+        q = payloads.first['queue']
+
+        conn.sadd('queues', q)
+
+        payloads.each do |entry|
+          to_push  = Sidekiq.dump_json(entry)
+          priority = entry['priority'] || 0
+          conn.zadd("queue:#{q}", priority, to_push)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/prioritized_queues/monkeypatches/web.rb

@@ -0,0 +1,15 @@
+require 'sinatra'
+
+module Sidekiq
+  class Web < Sinatra::Base
+    get "/queues/:name" do
+      halt 404 unless params[:name]
+      @count = (params[:count] || 25).to_i
+      @name = params[:name]
+      @queue = Sidekiq::Queue.new(@name)
+      (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
+      @messages = @messages.map {|msg, priority| Sidekiq::Job.new(msg, @name) }
+      erb :queue
+    end
+  end
+end

data/lib/sidekiq/prioritized_queues/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module PrioritizedQueues
+    VERSION = "0.1.0"
+  end
+end

data/sidekiq-prioritized_queues.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/prioritized_queues/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-prioritized_queues"
+  spec.version       = Sidekiq::PrioritizedQueues::VERSION
+  spec.authors       = ["Andre Medeiros"]
+  spec.email         = ["me@andremedeiros.info"]
+  spec.summary       = %q{Numeric priorities for queues on Sidekiq}
+  spec.description   = %q{Changes your queues from FIFO to numeric priority based ones.}
+  spec.homepage      = "https://github.com/publitas/sidekiq-prioritized_queues"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sidekiq", "~> 3.0"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
+end

data/test/fetch_test.rb

@@ -0,0 +1,24 @@
+require 'minitest_helper'
+
+module Sidekiq
+  module PrioritizedQueues
+    describe Fetch do
+      before do
+        Sidekiq.redis = REDIS
+        Sidekiq.redis { |c| c.flushdb }
+      end
+
+      it 'should fetch jobs in the right priority' do
+        client = Sidekiq::Client.new
+        client.push_bulk('class' => 'MockWorker', 'args' => [[20], [30], [10]])
+
+        fetcher = Sidekiq::PrioritizedQueues::Fetch.new(queues: ['default'])
+
+        [100, 200, 300].each do |priority|
+          msg = Sidekiq.load_json(fetcher.retrieve_work.message)
+          assert_equal priority, msg['priority']
+        end
+      end
+    end
+  end
+end

data/test/middleware_test.rb

@@ -0,0 +1,22 @@
+require 'minitest_helper'
+
+module Sidekiq
+  module PrioritizedQueues
+    describe Middleware do
+      before do
+        Sidekiq.redis = REDIS
+        Sidekiq.redis { |c| c.flushdb }
+      end
+
+      it 'should add the priority field to jobs' do
+        client = Sidekiq::Client.new
+        client.push('class' => 'MockWorker', 'args' => [10])
+
+        json = Sidekiq.redis { |c| c.zrange('queue:default', 0, 0) }.first
+        job  = Sidekiq.load_json(json)
+
+        assert_equal 100, job['priority']
+      end
+    end
+  end
+end

data/test/minitest_helper.rb

@@ -0,0 +1,18 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'sidekiq'
+require 'sidekiq/prioritized_queues'
+
+require 'minitest/autorun'
+
+REDIS = Sidekiq::RedisConnection.create(
+  url:       'redis://localhost/15',
+  namespace: 'sidekiq_prioritized_queues_test'
+)
+
+class MockWorker
+  include Sidekiq::Worker
+  sidekiq_options priority: -> (arg) { arg * 10 }
+
+  def perform(arg)
+  end
+end

data/test/monkeypatch_test.rb

@@ -0,0 +1,36 @@
+require 'minitest_helper'
+
+module Sidekiq
+  module PrioritizedQueues
+    describe 'Client Monkeypatch' do
+      before do
+        @redis = Minitest::Mock.new
+        def @redis.sadd(*); true; end
+        def @redis.with; yield self; end
+        def @redis.multi; [yield] * 2 if block_given?; end
+        Sidekiq.instance_variable_set(:@redis, @redis)
+        Sidekiq::Client.instance_variable_set(:@default, nil)
+      end
+
+      after do
+        Sidekiq.redis = REDIS
+        Sidekiq::Client.instance_variable_set(:@default, nil)
+      end
+
+      describe 'as an instance' do
+        it 'pushes jobs with the right score' do
+          @redis.expect :zadd, 1, ['queue:default', 50, String]
+          client = Sidekiq::Client.new
+          client.push('class' => 'MockWorker', 'args' => [5])
+          @redis.verify
+        end
+      end
+
+      it 'pushes jobs with the right score' do
+        @redis.expect :zadd, 1, ['queue:default', 20, String]
+        Sidekiq::Client.push('class' => 'MockWorker', 'args' => [2])
+        @redis.verify
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-prioritized_queues
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andre Medeiros
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Changes your queues from FIFO to numeric priority based ones.
+email:
+- me@andremedeiros.info
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq/prioritized_queues.rb
+- lib/sidekiq/prioritized_queues/fetch.rb
+- lib/sidekiq/prioritized_queues/middleware.rb
+- lib/sidekiq/prioritized_queues/monkeypatches.rb
+- lib/sidekiq/prioritized_queues/monkeypatches/api.rb
+- lib/sidekiq/prioritized_queues/monkeypatches/client.rb
+- lib/sidekiq/prioritized_queues/monkeypatches/web.rb
+- lib/sidekiq/prioritized_queues/version.rb
+- sidekiq-prioritized_queues.gemspec
+- test/fetch_test.rb
+- test/middleware_test.rb
+- test/minitest_helper.rb
+- test/monkeypatch_test.rb
+homepage: https://github.com/publitas/sidekiq-prioritized_queues
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Numeric priorities for queues on Sidekiq
+test_files:
+- test/fetch_test.rb
+- test/middleware_test.rb
+- test/minitest_helper.rb
+- test/monkeypatch_test.rb