sidekiq-ultimate 0.0.1.alpha

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 21eb153f8f9af214ece524d28299bb23e2a1918d90a8b99e41a63031eeb4037b
+  data.tar.gz: 9b930c25ddedb5177da9ee3bad6756c37f156355e9c639c00ecf8f5803016aeb
+SHA512:
+  metadata.gz: b38096795e63949611c87f0c42f35314ba7c0ec64b459fb91e6bb794e65aef2994c807a7d06873bedf92e78a343fedfcb95ece065478c5c1e0abf6a0b9d614c3
+  data.tar.gz: f36223cef6b03277f7090c0e5b1d3b33aeb70b7157c31205adbfc24d7a5ce4199499bd49ca215acfeac008a21d86973f321736326651e117594d0cd0fa031b9e

data/.gitignore

@@ -0,0 +1,10 @@
+/Gemfile.lock
+
+/.autoenv.zsh
+/.bundle
+/.yardoc
+/_yardoc
+/coverage
+/doc
+/pkg
+/tmp

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,46 @@
+AllCops:
+  TargetRubyVersion: 2.3
+  DisplayCopNames: true
+
+
+## Layout ######################################################################
+
+Layout/DotPosition:
+  EnforcedStyle: trailing
+
+Layout/IndentArray:
+  EnforcedStyle: consistent
+
+Layout/IndentHash:
+  EnforcedStyle: consistent
+
+
+## Metrics #####################################################################
+
+Metrics/BlockLength:
+  Exclude:
+    - "spec/**/*"
+
+
+## Style #######################################################################
+
+Style/BracesAroundHashParameters:
+  Enabled: false
+
+Style/HashSyntax:
+  EnforcedStyle: hash_rockets
+
+Style/RegexpLiteral:
+  EnforcedStyle: percent_r
+
+Style/RescueStandardError:
+  EnforcedStyle: implicit
+
+Style/SafeNavigation:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/YodaCondition:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,28 @@
+language: ruby
+sudo: false
+
+services:
+  - redis-server
+
+cache: bundler
+
+rvm:
+  - 2.3
+  - 2.4
+  - 2.5
+
+matrix:
+  fast_finish: true
+  include:
+    - rvm: 2.4
+      env: TEST_SUITE="rubocop"
+
+before_install:
+  - gem update --system
+  - gem --version
+  - gem install bundler --no-rdoc --no-ri
+  - bundle --version
+
+install: bundle install --without development doc
+
+script: bundle exec rake $TEST_SUITE

data/ARCHITECTURE.md

@@ -0,0 +1,115 @@
+# Decisions and Assumptions (AKA Architecture)
+
+This is a very proof of concept version of the architecture for reliable fetch
+strategy. IMplementation will be based on reliable queue pattern described in
+redis documentation. In short fetch will look like this:
+
+``` ruby
+COOLDOWN = 2
+IDENTITY = Object.new.tap { |o| o.extend Sidekiq::Util }.identity
+
+def retrieve
+  Sidekiq.redis do
+    queue_names.each do |name|
+      pending = "queue:#{name}"
+      inproc  = "inproc:#{IDENTITY}:#{name}"
+      job = redis.rpoplpush(pending, inproc)
+      return job if job
+    end
+  end
+
+  sleep COOLDOWN
+end
+```
+
+The above means that we will have inproc queue per queue and sidekiq server
+process. Naturally we will need a process that will monitor "orphan" queues.
+We will run such on each Sidekiq server in a `Concurrent::TimerTask` thread.
+We can easily check which sidekiq processes are currently alive with:
+
+``` ruby
+redis.exists(process_identity)
+```
+
+Sidekiq keeps it's own set of all known processes and clears it out upon first
+web view, so we need our own way to track all ever-running sidekiq process
+identities. So we can subscribe to `startup` event like so:
+
+``` ruby
+Sidekiq.on :startup do
+  Sidekiq.redis do |redis|
+    redis.sadd("ultimate:identities", IDENTITY)
+  end
+end
+```
+
+So now our casualties monitor can get all known identities and check which of
+them are still alive:
+
+``` ruby
+casualties = []
+
+Sidekiq.redis do |redis|
+  identities = redis.smembers("ultimate:identities")
+  heartbeats = redis.pipelined do
+    identities.each { |key| redis.exists(key) }
+  end
+
+  heartbeats.each_with_index do |exists, idx|
+    casualties << identities[idx] unless exists
+  end
+end
+```
+
+I want to put lost but found jobs back to the queue. But I want them to appear
+at the head of the pending queue so that thwey will be retried. So, I want some
+sort of LPOPRPUSH command, which does not exist, so we will use LUA script for
+that to guarantee atomic execution:
+
+``` lua
+local src = KEYS[1]
+local dst = KEYS[2]
+local val = redis.call("LPOP", src)
+
+if val then
+  redis.call("RPUSH", dst, val)
+end
+
+return val
+```
+
+So now our casualties monitor can start resurrecting them:
+
+``` ruby
+def resurrect
+  Sidekiq.redis do |redis|
+    casualties.each do |identity|
+      queue_names.each do |name|
+        src = "inproc:#{identity}:#{name}"
+        dst = "queue:#{name}"
+        loop while redis.eval(LPOPRPUSH, :keys => [src, dst])
+        redis.del(src)
+      end
+    end
+  end
+end
+```
+
+All good, but here's the problem: dead process could have differnet set of
+queues it was serving. So, instead of relying on `Sidekiq.options` we can
+save queues in the hash in redis, so our startup event will look like this:
+
+``` ruby
+Sidekiq.on :startup do
+  Sidekiq.redis do |redis|
+    queues = JSON.dump(Sidekiq.options[:queues].uniq)
+    redis.hmset("ultimate:identities", IDENTITY, queues)
+  end
+end
+```
+
+Now, to get casualties we will use `HKEYS` instead of `SMEMBERS`. But they have
+same complexity.
+
+In addition to the above we will be using redis-based locks to guarantee only
+one sidekiq process is handling resurrection at a time.

data/Gemfile

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+ruby RUBY_VERSION
+
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 0.52.0", :require => false
+
+group :development do
+  gem "guard",         :require => false
+  gem "guard-rspec",   :require => false
+  gem "guard-rubocop", :require => false
+  gem "pry",           :require => false
+end
+
+group :test do
+  gem "codecov",    :require => false
+  gem "simplecov",  :require => false
+end
+
+group :doc do
+  gem "redcarpet"
+  gem "yard"
+end
+
+# Specify your gem's dependencies in redis-prescription.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+guard :rspec, :cmd => "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end
+
+guard :rubocop do
+  watch(%r{.+\.rb$})
+  watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 SensorTower Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,116 @@
+# Sidekiq::Ultimate
+
+Sidekiq ultimate experience.
+
+---
+
+**WARNING**
+
+This ia an alpha/preview software. Lots of changes will be made and eventually
+it will overtake [sidekiq-throttled][] and will become truly ultimate sidekiq
+extension one will need. :D
+
+---
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "sidekiq-ultimate", ">= 0.0.1.alpha"
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-ultimate
+
+
+## Usage
+
+Add somewhere in your app's bootstrap (e.g. `config/initializers/sidekiq.rb` if
+you are using Rails):
+
+``` ruby
+require "sidekiq/ultimate"
+Sidekiq::Ultimate.setup!
+```
+
+---
+
+**NOTICE**
+
+Throttling is brought by [sidekiq-throttled][] and it's automatically set up
+by the command above - don't run `Sidekiq::Throttled.setup!` yourself.
+
+Thus look up it's README for throttling configuration details.
+
+---
+
+
+## Supported Ruby Versions
+
+This library aims to support and is [tested against][travis-ci] the following
+Ruby and Redis client versions:
+
+* Ruby
+  * 2.3.x
+  * 2.4.x
+  * 2.5.x
+
+* [redis-rb](https://github.com/redis/redis-rb)
+  * 4.x
+
+* [redis-namespace](https://github.com/resque/redis-namespace)
+  * 1.6
+
+
+If something doesn't work on one of these versions, it's a bug.
+
+This library may inadvertently work (or seem to work) on other Ruby versions,
+however support will only be provided for the versions listed above.
+
+If you would like this library to support another Ruby version or
+implementation, you may volunteer to be a maintainer. Being a maintainer
+entails making sure all tests run and pass on that implementation. When
+something breaks on your implementation, you will be responsible for providing
+patches in a timely fashion. If critical issues for a particular implementation
+exist at the time of a major release, support for that Ruby version may be
+dropped.
+
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies.
+Then, run `bundle exec rake spec` to run the tests with ruby-rb client.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+To release a new version, update the version number in `version.rb`, and then
+run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and tags, and push the `.gem` file to [rubygems.org][].
+
+
+## Contributing
+
+* Fork sidekiq-ultimate on GitHub
+* Make your changes
+* Ensure all tests pass (`bundle exec rake`)
+* Send a pull request
+* If we like them we'll merge them
+* If we've accepted a patch, feel free to ask for commit access!
+
+
+## Copyright
+
+Copyright (c) 2018 SensorTower Inc.<br>
+See [LICENSE.md][] for further details.
+
+
+[travis.ci]: http://travis-ci.org/sensortower/sidekiq-ultimate
+[rubygems.org]: https://rubygems.org
+[LICENSE.md]: https://github.com/sensortower/sidekiq-ultimate/blob/master/LICENSE.txt
+[sidekiq-throttled]: http://travis-ci.org/sensortower/sidekiq-throttled

data/Rakefile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new
+
+require "rubocop/rake_task"
+RuboCop::RakeTask.new
+
+if ENV["CI"]
+  task :default => :spec
+else
+  task :default => %i[rubocop spec]
+end

data/lib/sidekiq/ultimate.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "sidekiq/throttled"
+
+require "sidekiq/ultimate/version"
+
+module Sidekiq
+  # Sidekiq ultimate experience.
+  module Ultimate
+    class << self
+      # Sets up reliable throttled fetch and friends.
+      # @return [void]
+      def setup!
+        Sidekiq::Throttled::Communicator.instance.setup!
+        Sidekiq::Throttled::QueuesPauser.instance.setup!
+
+        Sidekiq.configure_server do |config|
+          require "sidekiq/ultimate/fetch"
+          Sidekiq::Ultimate::Fetch.setup!
+
+          require "sidekiq/throttled/middleware"
+          config.server_middleware do |chain|
+            chain.add Sidekiq::Throttled::Middleware
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/expirable_list.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "monitor"
+
+require "concurrent/utility/monotonic_time"
+
+module Sidekiq
+  module Ultimate
+    # List that tracks when elements were added and enumerates over those not
+    # older than `ttl` seconds ago.
+    #
+    # ## Implementation
+    #
+    # Internally list holds an array of arrays. Thus ecah element is a tuple of
+    # monotonic timestamp (when element was added) and element itself:
+    #
+    #     [
+    #       [ 123456.7890, "default" ],
+    #       [ 123456.7891, "urgent" ],
+    #       [ 123457.9621, "urgent" ],
+    #       ...
+    #     ]
+    #
+    # It does not deduplicates elements. Eviction happens only upon elements
+    # retrieval (see {#each}).
+    #
+    # @see http://ruby-concurrency.github.io/concurrent-ruby/Concurrent.html#monotonic_time-class_method
+    # @see https://ruby-doc.org/core/Process.html#method-c-clock_gettime
+    # @see https://linux.die.net/man/3/clock_gettime
+    #
+    # @private
+    class ExpirableList
+      include Enumerable
+
+      # Create a new ExpirableList instance.
+      #
+      # @param ttl [Float] elements time-to-live in seconds
+      def initialize(ttl)
+        @ttl = ttl.to_f
+        @arr = []
+        @mon = Monitor.new
+      end
+
+      # Pushes given element into the list.
+      #
+      # @params element [Object]
+      # @return [ExpirableList] self
+      def <<(element)
+        @mon.synchronize { @arr << [Concurrent.monotonic_time, element] }
+        self
+      end
+
+      # Evicts expired elements and calls the given block once for each element
+      # left, passing that element as a parameter.
+      #
+      # @yield [element]
+      # @return [Enumerator] if no block given
+      # @return [ExpirableList] self if block given
+      def each
+        return to_enum __method__ unless block_given?
+
+        # Evict expired elements
+        @mon.synchronize do
+          horizon = Concurrent.monotonic_time - @ttl
+          @arr.shift while @arr[0] && @arr[0][0] < horizon
+        end
+
+        @arr.dup.each { |element| yield element[1] }
+
+        self
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/fetch.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "sidekiq/throttled"
+
+require "sidekiq/ultimate/expirable_list"
+require "sidekiq/ultimate/queue_name"
+require "sidekiq/ultimate/resurrector"
+require "sidekiq/ultimate/unit_of_work"
+
+module Sidekiq
+  module Ultimate
+    # Throttled reliable fetcher implementing reliable queue pattern.
+    class Fetch
+      # Timeout to sleep between fetch retries in case of no job received,
+      # as well as timeout to wait for redis to give us something to work.
+      TIMEOUT = 2
+
+      def initialize(options)
+        @exhausted = ExpirableList.new(10 * TIMEOUT)
+
+        @strict = options[:strict] ? true : false
+        @queues = options[:queues].map { |name| QueueName.new(name) }
+
+        @queues.uniq! if @strict
+      end
+
+      def retrieve_work
+        work = retrieve
+
+        return unless work
+        return work unless work.throttled?
+
+        work.requeue_throttled
+        @exhausted << work.queue
+      end
+
+      def self.bulk_requeue(*)
+        # do nothing
+      end
+
+      def self.setup!
+        Sidekiq.options[:fetch] = self
+        Resurrector.setup!
+      end
+
+      private
+
+      def retrieve
+        Sidekiq.redis do |redis|
+          queues.each do |queue|
+            job = redis.rpoplpush(queue.pending, queue.inproc)
+            return UnitOfWork.new(queue, job) if job
+
+            @exhausted << queue
+          end
+        end
+
+        sleep TIMEOUT
+        nil
+      end
+
+      def queues
+        (@strict ? @queues : @queues.shuffle.uniq) - exhausted - paused_queues
+      end
+
+      def exhausted
+        @exhausted.to_a
+      end
+
+      def paused_queues
+        Sidekiq::Throttled::QueuesPauser.instance.
+          instance_variable_get(:@paused_queues).
+          map { |q| QueueName[q] }
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/lpoprpush.lua

@@ -0,0 +1,9 @@
+local src = KEYS[1]
+local dst = KEYS[2]
+local val = redis.call("LPOP", src)
+
+if val then
+  redis.call("RPUSH", dst, val)
+end
+
+return val

data/lib/sidekiq/ultimate/queue_name.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "sidekiq/util"
+
+module Sidekiq
+  module Ultimate
+    # Helper object that extend queue name string with redis keys.
+    #
+    # @private
+    class QueueName
+      # Regexp used to normalize (possibly) expanded queue name, e.g. the one
+      # that is returned upon redis BRPOP
+      QUEUE_PREFIX_RE = %r{.*queue:}
+      private_constant :QUEUE_PREFIX_RE
+
+      # Internal helper context.
+      Helper = Module.new { extend Sidekiq::Util }
+      private_constant :Helper
+
+      # Original stringified queue name.
+      #
+      # @example
+      #
+      #   queue_name.normalized # => "foobar"
+      #
+      # @return [String]
+      attr_reader :normalized
+      alias to_s normalized
+
+      # Create a new QueueName instance.
+      #
+      # @param normalized [#to_s] Normalized (without any namespaces or `queue:`
+      #   prefixes) queue name.
+      # @param identity [#to_s] Sidekiq process identity.
+      def initialize(normalized, identity: self.class.process_identity)
+        @normalized = -normalized.to_s
+        @identity   = -identity.to_s
+      end
+
+      # @!attribute [r] hash
+      #
+      #   A hash based on the normalized queue name.
+      #
+      #   @see https://ruby-doc.org/core/Object.html#method-i-hash
+      #   @return [Integer]
+      def hash
+        @hash ||= @normalized.hash
+      end
+
+      # @!attribute [r] pending
+      #
+      #   Redis key of queue list.
+      #
+      #   @example
+      #
+      #       queue_name.pending # => "queue:foobar"
+      #
+      #   @return [String]
+      def pending
+        @pending ||= -"queue:#{@normalized}"
+      end
+
+      # @!attribute [r] inproc
+      #
+      #   Redis key of in-process jobs list.
+      #
+      #   @example
+      #
+      #       queue_name.inproc # => "inproc:argentum:12345:a9b8c7d6e5f4:foobar"
+      #
+      #   @return [String]
+      def inproc
+        @inproc ||= -"inproc:#{@identity}:#{@normalized}"
+      end
+
+      # Check if `other` is the {QueueName} representing same queue.
+      #
+      # @example
+      #
+      #     QueueName.new("abc").eql? QueueName.new("abc") # => true
+      #     QueueName.new("abc").eql? QueueName.new("xyz") # => false
+      #
+      # @param other [Object]
+      # @return [Boolean]
+      def ==(other)
+        other.is_a?(self.class) && @normalized == other.normalized
+      end
+      alias eql? ==
+
+      # Returns human-friendly printable QueueName representation.
+      #
+      # @example
+      #
+      #     QueueName.new("foobar").inspect   # => QueueName["foobar"]
+      #     QueueName["queue:foobar"].inspect # => QueueName["foobar"]
+      #
+      # @return [String]
+      def inspect
+        "#{self.class}[#{@normalized.inspect}]"
+      end
+
+      # Returns new QueueName instance with normalized queue name. Use this
+      # when you're not sure if queue name is normalized or not (e.g. with
+      # queue name received as part of BRPOP command).
+      #
+      # @example
+      #
+      #   QueueName["ns:queue:foobar"].normalized # => "foobar"
+      #
+      # @param name [#to_s] Queue name
+      # @param kwargs (see #initialize for details on possible options)
+      # @return [QueueName]
+      def self.[](name, **kwargs)
+        new(name.to_s.sub(QUEUE_PREFIX_RE, ""), **kwargs)
+      end
+
+      def self.process_identity
+        Helper.identity
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/resurrector.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "redis/lockers"
+require "redis/prescription"
+
+module Sidekiq
+  module Ultimate
+    # Lost jobs resurrector.
+    module Resurrector
+      LPOPRPUSH = Redis::Prescription.read("#{__dir__}/lpoprpush.lua")
+      private_constant :LPOPRPUSH
+
+      MAIN_KEY = "ultimate:resurrector"
+      private_constant :MAIN_KEY
+
+      LOCK_KEY = "#{MAIN_KEY}:lock"
+      private_constant :LOCK_KEY
+
+      class << self
+        def setup!
+          ctulhu = Concurrent::TimerTask.new(:execution_interval => 5) do
+            resurrect!
+          end
+
+          Sidekiq.on(:startup) do
+            register_process!
+            ctulhu.execute
+          end
+
+          Sidekiq.on(:shutdown) { ctulhu.shutdown }
+        end
+
+        def resurrect!
+          lock do
+            casualties.each do |identity|
+              queues(identity).each { |queue| resurrect(queue) }
+              cleanup(identity)
+            end
+          end
+        end
+
+        private
+
+        def register_process!
+          Sidekiq.redis do |redis|
+            queues   = JSON.dump(Sidekiq.options[:queues].uniq)
+            identity = Object.new.tap { |o| o.extend Sidekiq::Util }.identity
+
+            redis.hset(MAIN_KEY, identity, queues)
+          end
+        end
+
+        def lock(&block)
+          Sidekiq.redis do |redis|
+            Redis::Lockers.acquire(redis, LOCK_KEY, :ttl => 30_000, &block)
+          end
+        end
+
+        def casualties
+          Sidekiq.redis do |redis|
+            casualties = []
+            identities = redis.hkeys(MAIN_KEY)
+
+            redis.pipelined { identities.each { |k| redis.exists k } }.
+              each_with_index { |v, i| casualties << identities[i] unless v }
+
+            casualties
+          end
+        end
+
+        def queues(identity)
+          Sidekiq.redis do |redis|
+            queues = redis.hget(MAIN_KEY, identity)
+
+            return [] unless queues
+
+            JSON.parse(queues).map do |q|
+              QueueName.new(q, :identity => identity)
+            end
+          end
+        end
+
+        def resurrect(queue)
+          Sidekiq.redis do |redis|
+            kwargs = { :keys => [queue.inproc, queue.pending] }
+            count  = 0
+
+            count += 1 while LPOPRPUSH.eval(redis, **kwargs)
+            redis.del(queue.inproc)
+          end
+        end
+
+        def cleanup(identity)
+          Sidekiq.redis { |redis| redis.hdel(MAIN_KEY, identity) }
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/unit_of_work.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "sidekiq/throttled"
+
+module Sidekiq
+  module Ultimate
+    # Job message envelope.
+    #
+    # @private
+    class UnitOfWork
+      # JSON payload
+      #
+      # @return [String]
+      attr_reader :job
+
+      # @param [QueueName] queue where job was pulled from
+      # @param [String] job JSON payload
+      def initialize(queue, job)
+        @queue = queue
+        @job   = job
+      end
+
+      # Pending jobs queue key name.
+      #
+      # @return [String]
+      def queue
+        @queue.pending
+      end
+
+      # Normalized `queue` name.
+      #
+      # @see QueueName#normalized
+      # @return [String]
+      def queue_name
+        @queue.normalized
+      end
+
+      # Remove job from the inproc list.
+      #
+      # Sidekiq calls this when it thinks jobs was performed with no mistakes.
+      #
+      # @return [void]
+      def acknowledge
+        Sidekiq.redis { |redis| redis.lrem(@queue.inproc, -1, @job) }
+      end
+
+      # We gonna resurrect jobs that were inside inproc queue upon process
+      # start, so no point in doing anything here.
+      #
+      # @return [void]
+      def requeue
+        # do nothing
+      end
+
+      # Pushes job back to the head of the queue, so that job won't be tried
+      # immediately after it was requeued (in most cases).
+      #
+      # @note This is triggered when job is throttled. So it is same operation
+      #   Sidekiq performs upon `Sidekiq::Worker.perform_async` call.
+      #
+      # @return [void]
+      def requeue_throttled
+        Sidekiq.redis do |redis|
+          redis.pipeline do
+            redis.lpush(@queue.pending, @job)
+            acknowledge
+          end
+        end
+      end
+
+      # Tells whenever job should be pushed back to queue (throttled) or not.
+      #
+      # @see Sidekiq::Throttled.throttled?
+      # @return [Boolean]
+      def throttled?
+        Sidekiq::Throttled.throttled?(@job)
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Ultimate
+    # Gem version.
+    VERSION = "0.0.1.alpha"
+  end
+end

data/sidekiq-ultimate.gemspec

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require "sidekiq/ultimate/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-ultimate"
+  spec.version       = Sidekiq::Ultimate::VERSION
+  spec.authors       = ["Alexey Zapparov"]
+  spec.email         = ["ixti@member.fsf.org"]
+
+  spec.summary       = "Sidekiq ultimate experience."
+  spec.description   = "Sidekiq ultimate experience."
+
+  spec.homepage      = "https://github.com/sensortower/sidekiq-ultimate"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "concurrent-ruby",    "~> 1.0"
+  spec.add_runtime_dependency "redis-lockers",      "~> 1.1"
+  spec.add_runtime_dependency "redis-prescription", "~> 1.0"
+  spec.add_runtime_dependency "sidekiq",            "~> 5.0"
+
+  # temporary couple this with sidekiq-throttled until it will be merged into
+  # this gem instead.
+  spec.add_runtime_dependency "sidekiq-throttled", "~> 0.8.2"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+
+  spec.required_ruby_version = "~> 2.3"
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-ultimate
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha
+platform: ruby
+authors:
+- Alexey Zapparov
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-02-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: redis-lockers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: redis-prescription
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq-throttled
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+description: Sidekiq ultimate experience.
+email:
+- ixti@member.fsf.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- ARCHITECTURE.md
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq/ultimate.rb
+- lib/sidekiq/ultimate/expirable_list.rb
+- lib/sidekiq/ultimate/fetch.rb
+- lib/sidekiq/ultimate/lpoprpush.lua
+- lib/sidekiq/ultimate/queue_name.rb
+- lib/sidekiq/ultimate/resurrector.rb
+- lib/sidekiq/ultimate/unit_of_work.rb
+- lib/sidekiq/ultimate/version.rb
+- sidekiq-ultimate.gemspec
+homepage: https://github.com/sensortower/sidekiq-ultimate
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: Sidekiq ultimate experience.
+test_files: []