sidekiq-priority_queue 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 23819a77a9986da48fe0e15a590bd948abf62cafa7d48f7ad8aa2d6129dbb476
+  data.tar.gz: cc8e7a8a99c095237a6b44c0a266753d98707c724e5af70b494a71b3a25476ac
+SHA512:
+  metadata.gz: '0884b842b53184df96c7e753d0d8d6b04621b2b5810a33e5798e95d76b13e09776ac4db62c520c0e822be0987b798deaa473e76898ecdd92b4095230b6b65501'
+  data.tar.gz: f1bb0dfb7a3048ce7ca80924916ac6817ed16262a2d98596f1da50db35292b2a2cb3287f0fcdb61a468f21f39d0dac5de3f8989b7bdc87515f728179fbb7576a

data/.github/workflows/tests.yml

@@ -0,0 +1,29 @@
+name: Tests
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+jobs:
+  tests:
+    name: Run tests
+    outputs:
+      job-status: ${{ job.status }}
+    runs-on: ubuntu-18.04
+    timeout-minutes: 10
+    services:
+      redis:
+        image: redis:alpine
+        ports:
+          - 6379:6379
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.3.1
+      - name: Set up correct version of Ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Install dependencies via Bundler
+        run: bundle install --jobs 4 --retry 3
+      - name: Run tests
+        run: bundle exec rake

data/.gitignore

@@ -0,0 +1,2 @@
+.vagrant
+/.idea/

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+gemspec
+
+
+group :test do
+  gem 'pry-byebug'
+  gem 'minitest'
+  gem 'rake'
+  gem 'simplecov'
+  gem 'rack-test'
+end

data/Gemfile.lock

@@ -0,0 +1,50 @@
+PATH
+  remote: .
+  specs:
+    sidekiq-priority_queue (1.0.2)
+      sidekiq (>= 6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (11.1.3)
+    coderay (1.1.3)
+    connection_pool (2.2.3)
+    docile (1.3.4)
+    method_source (1.0.0)
+    minitest (5.14.3)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rake (13.0.3)
+    redis (4.2.5)
+    sidekiq (6.1.3)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest
+  pry-byebug
+  rack-test
+  rake
+  sidekiq-priority_queue!
+  simplecov
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,5 @@
+Copyright (c) ChartMogul Ltd
+
+Sidekiq-priority_queue is an Open Source project licensed under the terms of
+the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
+for license text.

data/README.md

@@ -0,0 +1,64 @@
+Sidekiq Priority Queue
+==============
+Extends Sidekiq with support for queuing jobs with a fine grained priority and emulating multiple queues using a single Redis sorted set, ideal for multi-tenant applications.
+
+The standard Sidekiq setup performs really well using Redis lists, but lists can only be strict FIFO queues, which can be hugely problematic when they process slowly and one user may need to wait hours behind a backlog of jobs.
+
+Sidekiq Priority Queue offers a plug-in solution retaining the simplicity and performance of Sidekiq. The priority queue is a building block for emulating sub-queues (per tenant or user) by de-prioritising jobs according to how many jobs are already in this sub-queue.
+
+Sources of inspiration are naturally Sidekiq itself, the fantastic Redis documentation, and https://github.com/gocraft/work
+
+Installation
+-----------------
+
+    gem install sidekiq-priority_queue
+
+Configuration
+-----------------   
+```
+Sidekiq.configure_server do |config|
+  config.options[:fetch] = Sidekiq::PriorityQueue::Fetch
+end
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::PriorityQueue::Client
+  end
+end
+```
+Usage
+-----------------   
+```
+class Worker
+  include Sidekiq::Worker
+  sidekiq_options priority: 1000
+end
+```
+Alternatively, you can split jobs into subqueues (via a proc) which are deprioritised based on the subqueue size:
+```
+class Worker
+  include Sidekiq::Worker
+
+  # args[0] will take the `user_id` argument below, and assign priority dynamically.
+  sidekiq_options subqueue: ->(args){ args[0] }
+
+  def perform(user_id, other_args)
+    # do jobs
+  end
+end
+```
+
+Testing
+-----------------
+For example in your `spec/rails_helper.rb` you can include this line:
+```ruby
+require 'sidekiq/priority_queue/testing'
+```
+next to the call to `Sidekiq::Testing.inline!`. It disables the feature for testing and falls back to `inline`/`fake` modes.
+If you accidentally require this in production code, it will likewise fall back to normal Sidekiq scheduling.
+
+Development
+-----------------
+- Run `docker-compose up -d` to start up a temporary redis instance.
+- Run `bundle install` to install dependencies.
+- Run the tests with `bundle exec rake`

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.warning = true
+  test.pattern = 'test/**/test_*.rb'
+end
+
+task :default => :test

data/bin/sidekiqload

@@ -0,0 +1,159 @@
+#!/usr/bin/env ruby
+
+# Copyright (c) Contributed Systems LLC, ChartMogul Ltd
+
+# Sidekiq-priority_queue is an Open Source project licensed under the terms of
+# the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
+# for license text.
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+
+#require 'ruby-prof'
+Bundler.require(:default)
+
+require 'sidekiq/cli'
+require 'sidekiq/launcher'
+require 'sidekiq/priority_queue'
+
+include Sidekiq::Util
+
+Sidekiq.configure_server do |config|
+  #config.options[:concurrency] = 1
+  config.redis = { db: 13 }
+  config.options[:queues] << 'default'
+  config.logger.level = Logger::ERROR
+  config.average_scheduled_poll_interval = 2
+  config.options[:fetch] = Sidekiq::PriorityQueue::Fetch
+end
+
+Sidekiq.configure_client do |config|
+    config.client_middleware do |chain|
+        chain.add Sidekiq::PriorityQueue::Client
+    end
+end
+
+class LoadWorker
+  include Sidekiq::Worker
+  sidekiq_options retry: 1, subqueue: ->(args){ args[0] }
+  sidekiq_retry_in do |x|
+    1
+  end
+
+  def perform(idx)
+    #raise idx.to_s if idx % 100 == 1
+  end
+end
+
+# brew tap shopify/shopify
+# brew install toxiproxy
+# gem install toxiproxy
+#require 'toxiproxy'
+# simulate a non-localhost network for realer-world conditions.
+# adding 1ms of network latency has an ENORMOUS impact on benchmarks
+#Toxiproxy.populate([{
+    #"name": "redis",
+    #"listen": "127.0.0.1:6380",
+    #"upstream": "127.0.0.1:6379"
+#}])
+
+self_read, self_write = IO.pipe
+%w(INT TERM TSTP TTIN).each do |sig|
+  begin
+    trap sig do
+      self_write.puts(sig)
+    end
+  rescue ArgumentError
+    puts "Signal #{sig} not supported"
+  end
+end
+
+Sidekiq.redis {|c| c.flushdb}
+def handle_signal(launcher, sig)
+  Sidekiq.logger.debug "Got #{sig} signal"
+  case sig
+  when 'INT'
+    # Handle Ctrl-C in JRuby like MRI
+    # http://jira.codehaus.org/browse/JRUBY-4637
+    raise Interrupt
+  when 'TERM'
+    # Heroku sends TERM and then waits 10 seconds for process to exit.
+    raise Interrupt
+  when 'TSTP'
+    Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+    launcher.quiet
+  when 'TTIN'
+    Thread.list.each do |thread|
+      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
+      if thread.backtrace
+        Sidekiq.logger.warn thread.backtrace.join("\n")
+      else
+        Sidekiq.logger.warn "<no backtrace available>"
+      end
+    end
+  end
+end
+
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+
+Sidekiq.redis do |con|
+count = 100_000
+
+  count.times do |idx|
+    #TODO why does Sidekiq::Client.push not work as expected?
+    con.zadd('priority-queue:default', idx, { 'class' => LoadWorker, 'args' => [idx] }.to_json)
+  end
+end
+Sidekiq.logger.error "Created #{ Sidekiq::PriorityQueue::Queue.new().size } jobs"
+
+Monitoring = Thread.new do
+  watchdog("monitor thread") do
+    while true
+      sleep 1
+      qsize, retries = Sidekiq.redis do |conn|
+        conn.pipelined do
+          conn.zcard "priority-queue:default"
+          conn.zcard "retry"
+        end
+      end.map(&:to_i)
+      total = qsize + retries
+      #GC.start
+      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+      if total == 0
+        Sidekiq.logger.error("Done")
+        exit(0)
+      end
+    end
+  end
+end
+
+begin
+  #RubyProf::exclude_threads = [ Monitoring ]
+  #RubyProf.start
+  fire_event(:startup)
+  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+    launcher = Sidekiq::Launcher.new(Sidekiq.options)
+    launcher.run
+
+    while readable_io = IO.select([self_read])
+      signal = readable_io.first[0].gets.strip
+      handle_signal(launcher, signal)
+    end
+  #end
+rescue SystemExit => e
+  #Sidekiq.logger.error("Profiling...")
+  #result = RubyProf.stop
+  #printer = RubyProf::GraphHtmlPrinter.new(result)
+  #printer.print(File.new("output.html", "w"), :min_percent => 1)
+  # normal
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/docker-compose.yml

@@ -0,0 +1,7 @@
+---
+version: '3.3'
+services:
+  redis:
+    image: redis:alpine
+    ports:
+      - 6379:6379

data/lib/sidekiq/priority_queue.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require 'sidekiq/priority_queue/api'
+require 'sidekiq/priority_queue/client'
+require 'sidekiq/priority_queue/combined_fetch'
+require 'sidekiq/priority_queue/fetch'
+require 'sidekiq/priority_queue/reliable_fetch'
+require 'sidekiq/priority_queue/scripts'
+require 'sidekiq/priority_queue/web'
+
+module Sidekiq
+  module PriorityQueue
+  end
+end

data/lib/sidekiq/priority_queue/api.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+require 'sidekiq/api'
+
+
+module Sidekiq
+  module PriorityQueue
+    class Queue
+      include  Enumerable
+
+      attr_reader :name
+
+      def initialize(name='default')
+        @name = name
+        @rname = "priority-queue:#{name}"
+      end
+
+      def size
+        Sidekiq.redis { |con| con.zcard(@rname) }
+      end
+
+      def each
+        initial_size = size
+        deleted_size = 0
+        page = 0
+        page_size = 50
+
+        while true do
+          range_start = page * page_size - deleted_size
+          range_end   = range_start + page_size - 1
+          entries = Sidekiq.redis do |conn|
+            conn.zrange @rname, range_start, range_end, withscores: true
+          end
+          break if entries.empty?
+          page += 1
+          entries.each do |entry, priority|
+            yield Job.new(entry, @name, priority)
+          end
+          deleted_size = initial_size - size
+        end
+      end
+
+      def self.all
+         Sidekiq.redis { |con| con.smembers('priority-queues') }
+          .map{ |key| key.gsub('priority-queue:', '') }
+          .sort
+          .map { |q| Queue.new(q) }
+      end
+
+    end
+
+    SubqueueCount = Struct.new(:name, :size)
+
+    class Job < Sidekiq::Job
+
+      attr_reader :priority
+      attr_reader :subqueue
+
+      def initialize(item, queue_name = nil, priority = nil)
+        @args = nil
+        @value = item
+        @item = item.is_a?(Hash) ? item : parse(item)
+        @queue = queue_name || @item['queue']
+        @subqueue = @item['subqueue']
+        @priority = priority
+      end
+
+      def delete
+        count = Sidekiq.redis do |conn|
+          conn.zrem("priority-queue:#{@queue}", @value)
+        end
+        count != 0
+      end
+    end
+  end
+end

data/lib/sidekiq/priority_queue/client.rb

@@ -0,0 +1,68 @@
+module Sidekiq
+  module PriorityQueue
+    class Client
+
+      # inserted into Sidekiq's Client as middleware
+      def call(worker_class, item, queue, redis_pool)
+        if item['priority']
+          sadd('priority-queues', queue)
+          zadd(queue, item['priority'], item)
+          return item['jid']
+        elsif item['subqueue']
+          # replace the proc with what it returns
+          sadd('priority-queues', queue)
+          item['subqueue'] = resolve_subqueue(item['subqueue'], item['args'])
+          priority = fetch_and_add(queue, item['subqueue'], item)
+          zadd(queue, priority, item)
+          return item['jid']
+        else
+          # continue pushing the normal Sidekiq way
+          yield
+        end
+      end
+
+      private
+
+      def resolve_subqueue(subqueue, job_args)
+        return subqueue unless subqueue.respond_to?(:call)
+
+        subqueue.call(job_args)
+      end
+
+      def zadd(queue, score, item)
+        Sidekiq.redis do |conn|
+          queue = "priority-queue:#{queue}"
+          conn.zadd(queue, score, item.to_json)
+          return item
+        end
+      end
+
+      def sadd(set, member)
+        Sidekiq.redis do |conn|
+          conn.sadd(set,member)
+        end
+      end
+
+      def fetch_and_add(queue, subqueue, item)
+        Sidekiq.redis do |conn|
+          priority = conn.zincrby("priority-queue-counts:#{queue}", 1, subqueue)
+        end
+      end
+    end
+  end
+end
+
+Sidekiq::Client.class_eval do
+  def push(item)
+    normed = normalize_item(item)
+    payload = process_single(item['class'], normed)
+
+    # if payload is a JID because the middleware already pushed then just return the JID
+    return payload if payload.is_a?(String)
+
+    if payload
+      raw_push([payload])
+      payload['jid']
+    end
+  end
+end

data/lib/sidekiq/priority_queue/combined_fetch.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module PriorityQueue
+    class CombinedFetch
+      attr_reader :fetches
+
+      def initialize(fetches = [])
+        @fetches = fetches
+      end
+
+      def retrieve_work
+        fetches.each do |fetch|
+          work = fetch.retrieve_work
+          return work if work
+        end
+      end
+
+      def self.configure(&block)
+        combined_fetch = self.new
+        yield combined_fetch
+
+        combined_fetch
+      end
+
+      def add(fetch)
+        fetches << fetch
+      end
+
+      def bulk_requeue(inprogress, options)
+        # ReliableFetch#bulk_equeue ignores inprogress, so it's safe to call both
+        fetches.each do |f|
+          if [Fetch, ReliableFetch].any? { |klass| f.is_a?(klass) }
+            jobs_to_requeue = inprogress.select{|uow| uow.queue.start_with?('priority-queue:') }
+            f.bulk_requeue(jobs_to_requeue, options)
+          else
+            jobs_to_requeue = inprogress.reject{|uow| uow.queue.start_with?('priority-queue:') }
+            f.bulk_requeue(jobs_to_requeue, options)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/priority_queue/fetch.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+require 'sidekiq'
+
+module Sidekiq
+  module PriorityQueue
+    class Fetch
+
+      UnitOfWork = Struct.new(:queue, :job) do
+        def acknowledge
+          Sidekiq.redis do |conn|
+            unless subqueue.nil?
+              count = conn.zincrby(subqueue_counts, -1, subqueue)
+              conn.zrem(subqueue_counts, subqueue) if count < 1
+            end
+          end
+        end
+
+        def queue_name
+          queue.sub(/.*queue:/, '')
+        end
+
+        def subqueue
+          @parsed_job ||= JSON.parse(job)
+          @parsed_job['subqueue']
+        end
+
+        def subqueue_counts
+          "priority-queue-counts:#{queue_name}"
+        end
+
+        def requeue
+          Sidekiq.redis do |conn|
+            conn.zadd(queue, 0, job)
+          end
+        end
+      end
+
+      def initialize(options)
+        @strictly_ordered_queues = !!options[:strict]
+        @queues = options[:queues].map { |q| "priority-queue:#{q}" }
+        @queues = @queues.uniq if @strictly_ordered_queues
+      end
+
+      def retrieve_work
+        work = @queues.detect{ |q| job = zpopmin(q); break [q,job] if job }
+        UnitOfWork.new(*work) if work
+      end
+
+      def zpopmin(queue)
+        Sidekiq.redis do |con|
+          @script_sha ||= con.script(:load, Sidekiq::PriorityQueue::Scripts::ZPOPMIN)
+          con.evalsha(@script_sha, [queue])
+        end
+      end
+
+      def queues_cmd
+        if @strictly_ordered_queues
+          @queues
+        else
+          @queues.shuffle.uniq
+        end
+      end
+
+      def bulk_requeue(inprogress, options)
+        return if inprogress.empty?
+
+        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+        jobs_to_requeue = {}
+        inprogress.each do |unit_of_work|
+          jobs_to_requeue[unit_of_work.queue] ||= []
+          jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
+        end
+
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            jobs_to_requeue.each do |queue, jobs|
+              conn.zadd(queue, jobs.map{|j| [0,j] })
+            end
+          end
+        end
+        Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+      rescue => ex
+        Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+      end
+    end
+  end
+end

data/lib/sidekiq/priority_queue/reliable_fetch.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+require 'sidekiq'
+
+module Sidekiq
+  module PriorityQueue
+    class ReliableFetch
+
+      UnitOfWork = Struct.new(:queue, :job, :wip_queue) do
+        def acknowledge
+          Sidekiq.redis do |conn|
+            conn.srem(wip_queue, job)
+            unless subqueue.nil?
+              count = conn.zincrby(subqueue_counts, -1, subqueue)
+              conn.zrem(subqueue_counts, subqueue) if count < 1
+            end
+          end
+        end
+
+        def queue_name
+          queue.sub(/.*queue:/, '')
+        end
+
+        def subqueue
+          @parsed_job ||= JSON.parse(job)
+          @parsed_job['subqueue']
+        end
+
+        def subqueue_counts
+          "priority-queue-counts:#{queue_name}"
+        end
+
+        def requeue
+          # Nothing needed. Jobs are in WIP queue.
+        end
+      end
+
+      def initialize(options)
+        @strictly_ordered_queues = !!options[:strict]
+        @queues = options[:queues].map { |q| "priority-queue:#{q}" }
+        @queues = @queues.uniq if @strictly_ordered_queues
+        @process_index = options[:index] || ENV['PROCESS_INDEX']
+      end
+
+      def retrieve_work
+        work = @queues.detect do |q|
+          job = zpopmin_sadd(q, wip_queue(q));
+          break [q,job] if job
+        end
+        UnitOfWork.new(*work, wip_queue(work.first)) if work
+      end
+
+      def wip_queue(q)
+        "#{q}_#{Socket.gethostname}_#{@process_index}"
+      end
+
+      def zpopmin_sadd(queue, wip_queue)
+        Sidekiq.redis do |con|
+          @script_sha ||= con.script(:load, Sidekiq::PriorityQueue::Scripts::ZPOPMIN_SADD)
+          con.evalsha(@script_sha, [queue, wip_queue])
+        end
+      end
+
+      def spop(wip_queue)
+        Sidekiq.redis{ |con| con.spop(wip_queue) }
+      end
+
+      def queues_cmd
+        if @strictly_ordered_queues
+          @queues
+        else
+          @queues.shuffle.uniq
+        end
+      end
+
+      def bulk_requeue(_inprogress, options)
+        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+        process_index = options[:index] || ENV['PROCESS_INDEX']
+        self.class.requeue_wip_jobs(options[:queues], process_index)
+      end
+
+      def self.resume_wip_jobs(queues, process_index)
+        Sidekiq.logger.debug { "Re-queueing WIP jobs" }
+        process_index ||= ENV['PROCESS_INDEX']
+        requeue_wip_jobs(queues, process_index)
+      end
+
+      Sidekiq.configure_server do |config|
+        config.on(:startup) do
+          if reliable_fetch_active?(config)
+            Sidekiq::PriorityQueue::ReliableFetch.resume_wip_jobs(config.options[:queues], config.options[:index])
+          end
+        end
+      end
+
+      private
+
+      def self.reliable_fetch_active?(config)
+        return true if config.options[:fetch].is_a?(Sidekiq::PriorityQueue::ReliableFetch)
+        return config.options[:fetch].is_a?(Sidekiq::PriorityQueue::CombinedFetch) &&
+          config.options[:fetch].fetches.any? { |f| f.is_a?(Sidekiq::PriorityQueue::ReliableFetch) }
+      end
+
+      def self.requeue_wip_jobs(queues, index)
+        jobs_to_requeue = {}
+        Sidekiq.redis do |conn|
+          queues.map { |q| "priority-queue:#{q}" }.each do |q|
+            wip_queue = "#{q}_#{Socket.gethostname}_#{index}"
+            jobs_to_requeue[q] = []
+            while job = conn.spop(wip_queue) do
+              jobs_to_requeue[q] << job
+            end
+          end
+
+          conn.pipelined do
+            jobs_to_requeue.each do |queue, jobs|
+              return unless jobs.size > 0
+              conn.zadd(queue, jobs.map{|j| [0,j] })
+            end
+          end
+        end
+        Sidekiq.logger.info("Pushed #{ jobs_to_requeue.map{|q| q.size }.reduce(:+) } jobs back to Redis")
+      rescue => ex
+        Sidekiq.logger.warn("Failed to requeue #{ jobs_to_requeue.map{|q| q.size }.reduce(:+) } jobs: #{ex.message}")
+      end
+    end
+  end
+end

data/lib/sidekiq/priority_queue/scripts.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+
+module Sidekiq
+  module PriorityQueue
+    module Scripts
+
+      ZPOPMIN = %q(
+        local resp = redis.call('zrange', KEYS[1], '0', '0')
+        if (resp[1] ~= nil) then
+          local val = resp[# resp]
+          redis.call('zrem', KEYS[1], val)
+          return val
+        else
+          return false
+        end
+      )
+
+      ZPOPMIN_SADD = %q(
+        local resp = redis.call('zrange', KEYS[1], '0', '0')
+        if (resp[1] ~= nil) then
+          local val = resp[# resp]
+          redis.call('zrem', KEYS[1], val)
+          redis.call('sadd', KEYS[2], val)
+          return val
+        else
+          return false
+        end
+      )
+
+    end
+  end
+end

data/lib/sidekiq/priority_queue/testing.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Don't require in production code.
+# This disables the middleware and falls back to normal push, meaning in tests it will use inline/fake mode.
+# Prioritization doesn't make any sense in inline or fake tests anyway.
+module Sidekiq
+  module PriorityQueue
+    module TestingClient
+      def call(worker_class, item, queue, redis_pool)
+        testing_verify_subqueue(item) if item['subqueue'] && !item['priority']
+        yield # continue pushing the normal Sidekiq way
+      end
+
+      # Help testing the lambda; raise in case it's invalid.
+      def testing_verify_subqueue(item)
+        subqueue = resolve_subqueue(item['subqueue'], item['args'])
+        serialized = "#{subqueue}"
+
+        raise "subqueue shouldn't be nil" if subqueue.nil?
+        raise "subqueue shouldn't be empty" if serialized == ""
+      end
+    end
+
+    Sidekiq::PriorityQueue::Client.prepend TestingClient
+  end
+end

data/lib/sidekiq/priority_queue/web.rb

@@ -0,0 +1,42 @@
+require 'sidekiq/web'
+
+
+module Sidekiq::PriorityQueue
+  module Web
+
+    ROOT = File.expand_path('../web', __FILE__)
+
+    def self.registered(app)
+      app.tabs['Priority Queues'] = 'priority_queues'
+
+      app.get '/priority_queues' do
+        @queues = Queue.all
+        render(:erb, File.read("#{ROOT}/views/priority_queues.erb"))
+      end
+
+      app.get '/priority_queues/:name' do
+        @name = route_params[:name]
+        halt(404) unless @name
+
+        @count = (params['count'] || 25).to_i
+        @queue = Sidekiq::Queue.new(@name)
+        (@current_page, @total_size, @messages) = page("priority-queue:#{@name}", params['page'], @count)
+        @subqueue_counts = Sidekiq.redis do |con|
+          con.zrevrange("priority-queue-counts:#{@name}", 0, params['subqueue_count'] || 10, withscores: true)
+        end.map { |name, count| SubqueueCount.new(name, count) }
+
+        @messages = @messages.map{ |msg| Job.new(msg.first, @name, msg.last) }
+        render(:erb, File.read("#{ROOT}/views/priority_queue.erb"))
+      end
+
+      app.post "/priority_queues/:name/delete" do
+        name = route_params[:name]
+        Job.new(params['key_val'], name).delete
+        redirect_with_query("#{root_path}priority_queues/#{CGI.escape(name)}")
+      end
+
+    end
+  end
+end
+
+::Sidekiq::Web.register Sidekiq::PriorityQueue::Web

data/lib/sidekiq/priority_queue/web/views/_paging.erb

@@ -0,0 +1,24 @@
+
+<% if @total_size > @count %>
+  <ul class="pagination pull-right flip">
+    <li class="<%= 'disabled' if @current_page == 1 %>">
+      <a href="<%= url %>?page=1">&laquo;</a>
+    </li>
+    <% if @current_page > 1 %>
+      <li>
+        <a href="<%= url %>?<%= qparams(page: @current_page - 1) %>"><%= @current_page - 1 %></a>
+      </li>
+    <% end %>
+    <li class="disabled">
+      <a href="<%= url %>?<%= qparams(page: @current_page) %>"><%= @current_page %></a>
+    </li>
+    <% if @total_size > @current_page * @count %>
+      <li>
+        <a href="<%= url %>?<%= qparams(page: @current_page + 1) %>"><%= @current_page + 1 %></a>
+      </li>
+    <% end %>
+    <li class="<%= 'disabled' if @total_size <= @current_page * @count %>">
+      <a href="<%= url %>?<%= qparams(page: (@total_size.to_f / @count).ceil) %>">&raquo;</a>
+    </li>
+  </ul>
+<% end %>

data/lib/sidekiq/priority_queue/web/views/priority_queue.erb

@@ -0,0 +1,74 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>
+      <%= t('CurrentMessagesInQueue', :queue => h(@name)) %>
+
+      <span class="badge badge-secondary"><%= number_with_delimiter(@total_size) %></span>
+    </h3>
+  </div>
+  <div class="col-sm-4 pull-right flip">
+    <%= erb :_paging, locals: { url: "#{root_path}priority_queues/#{CGI.escape(@name)}" } %>
+  </div>
+</header>
+<div class="row">
+  <div class="col-sm-12">
+    <h3>Biggest subqueues</h3>
+
+    <summary>
+      <details>
+        <div class="table_container">
+          <table class="table table-hover table-bordered table-striped">
+            <thead>
+              <th>Subqueue name</th>
+              <th>Subqueue size</th>
+            </thead>
+            <% @subqueue_counts.each do |subqueue_count| %>
+              <tr>
+                <td><%= subqueue_count.name %></td>
+                <td><%= subqueue_count.size %></td>
+              </tr>
+            <% end %>
+          </table>
+        </div>
+      </details>
+    </summary>
+
+  </div>
+</div>
+<hr />
+<div class="table_container">
+  <table class="queue table table-hover table-bordered table-striped">
+    <thead>
+      <th><%= t('Job') %></th>
+      <th><%= t('Arguments') %></th>
+      <th><%= t('Priority') %></th>
+      <th><%= t('Subqueue') %></th>
+      <th></th>
+    </thead>
+    <% @messages.each_with_index do |msg, index| %>
+      <tr>
+        <td><%= h(msg.display_class) %></td>
+        <td>
+          <% a = msg.display_args %>
+          <% if a.inspect.size > 100 %>
+            <span class="worker_<%= index %>"><%= h(a.inspect[0..100]) + "... " %></span>
+            <button data-toggle="collapse" data-target=".worker_<%= index %>" class="btn btn-default btn-xs"><%= t('ShowAll') %></button>
+            <div class="toggle worker_<%= index %>"><%= display_args(a) %></div>
+          <% else %>
+            <%= display_args(msg.display_args) %>
+          <% end %>
+        </td>
+        <td><%= msg.priority %></td>
+        <td><%= msg.subqueue %></td>
+        <td>
+          <form action="<%= root_path %>priority_queues/<%= CGI.escape(@name) %>/delete" method="post">
+            <%= csrf_tag %>
+            <input name="key_val" value="<%= h msg.value %>" type="hidden" />
+            <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+          </form>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+</div>
+<%= erb :_paging, locals: { url: "#{root_path}priority_queues/#{@name}" } %>

data/lib/sidekiq/priority_queue/web/views/priority_queues.erb

@@ -0,0 +1,27 @@
+<h3><%= t('Queues') %></h3>
+
+<div class="table_container">
+  <table class="queues table table-hover table-bordered table-striped table-white">
+    <thead>
+      <th><%= t('Queue') %></th>
+      <th><%= t('Size') %></th>
+      <th><%= t('Latency') %></th>
+      <th><%= t('Actions') %></th>
+    </thead>
+    <% @queues.each do |queue| %>
+      <tr>
+        <td>
+          <a href="<%= root_path %>priority_queues/<%= CGI.escape(queue.name) %>"><%= h queue.name %></a>
+        </td>
+        <td><%= queue.size %> </td>
+        <td><%# number_with_delimiter(queue.latency.round(2)) %> </td>
+        <td class="delete-confirm">
+          <form action="<%=root_path %>priority_queues/<%= CGI.escape(queue.name) %>" method="post">
+            <%= csrf_tag %>
+            <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSureDeleteQueue', :queue => h(queue.name)) %>" />
+          </form>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+</div>

data/sidekiq-priority_queue.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name        = 'sidekiq-priority_queue'
+  s.version     = '1.0.2'
+  s.date        = '2018-07-31'
+  s.summary     = "Priority Queuing for Sidekiq"
+  s.description = "An extension for Sidekiq allowing jobs in a single queue to be executed by a priority score rather than FIFO"
+  s.authors     = ["Jacob Matthews", "Petr Kopac"]
+  s.email       = 'petr@chartmogul.com'
+  s.files       = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|pkg)/}) }
+  s.homepage    = 'https://github.com/chartmogul/sidekiq-priority_queue'
+  s.license     = 'MIT'
+  s.required_ruby_version = '>= 2.5.0'
+
+  s.add_dependency 'sidekiq', '>= 6'
+  s.add_development_dependency 'minitest', '~> 5.10', '>= 5.10.1'
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-priority_queue
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- Jacob Matthews
+- Petr Kopac
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.10.1
+description: An extension for Sidekiq allowing jobs in a single queue to be executed
+  by a priority score rather than FIFO
+email: petr@chartmogul.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/tests.yml"
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/sidekiqload
+- docker-compose.yml
+- lib/sidekiq/priority_queue.rb
+- lib/sidekiq/priority_queue/api.rb
+- lib/sidekiq/priority_queue/client.rb
+- lib/sidekiq/priority_queue/combined_fetch.rb
+- lib/sidekiq/priority_queue/fetch.rb
+- lib/sidekiq/priority_queue/reliable_fetch.rb
+- lib/sidekiq/priority_queue/scripts.rb
+- lib/sidekiq/priority_queue/testing.rb
+- lib/sidekiq/priority_queue/web.rb
+- lib/sidekiq/priority_queue/web/views/_paging.erb
+- lib/sidekiq/priority_queue/web/views/priority_queue.erb
+- lib/sidekiq/priority_queue/web/views/priority_queues.erb
+- sidekiq-priority_queue.gemspec
+homepage: https://github.com/chartmogul/sidekiq-priority_queue
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Priority Queuing for Sidekiq
+test_files: []