atomic-sidekiq 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1af076c37beeb362724591b736eca75821223033
+  data.tar.gz: a8ffb7fcfafa8ee3f70e24e13b728703ad55e4c8
+SHA512:
+  metadata.gz: 6ae93a470419fac19638dd02a0f92f7f6972a9129ac11fba26265062a3ab186dfb1962ed4718fdce84cf2a2c1ef0ecae07d0fb6df79d6316d78224cd33d6b755
+  data.tar.gz: 4ca5974dc84bd4d8fa1b8efbbf4bb5a862022039cc8336b149ba0287665324d5565846da977b21eaf1fd9574d666f03f455f15643d7713fa5168018946a53004

data/.gitignore

@@ -0,0 +1 @@
+.byebug_history

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper -I ./lib

data/Dockerfile

@@ -0,0 +1,22 @@
+# Choose the official Ruby 2.3.4 image as our starting point
+FROM ruby:2.5.1
+
+ENV LC_ALL en_US.UTF-8
+
+# install locked bundler version (1.16.1)
+RUN gem install bundler -v 1.16.1
+ENV BUNDLE_PATH=/bundle BUNDLE_JOBS=4
+
+# Set up working directory
+ENV APP_HOME /sidekiq-atomic
+
+RUN mkdir $APP_HOME
+
+WORKDIR $APP_HOME
+
+ADD Gemfile .
+ADD Gemfile.lock .
+
+RUN bundle install
+
+ADD . $APP_HOME

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: .
+  specs:
+    atomic-sidekiq (0.0.0)
+      sidekiq (~> 5.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    byebug (10.0.2)
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
+    diff-lcs (1.3)
+    parallel (1.12.1)
+    parser (2.5.0.5)
+      ast (~> 2.4.0)
+    powerpack (0.1.1)
+    rack (2.0.4)
+    rack-protection (2.0.1)
+      rack
+    rainbow (3.0.0)
+    rake (11.3.0)
+    redis (4.0.1)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.1)
+    rubocop (0.54.0)
+      parallel (~> 1.10)
+      parser (>= 2.5)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.9.0)
+    sidekiq (5.1.2)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.5, < 5)
+    timecop (0.9.1)
+    unicode-display_width (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  atomic-sidekiq!
+  bundler (~> 1.12)
+  byebug (~> 10.0)
+  rake (~> 11.3)
+  rspec (~> 3.6)
+  rubocop (~> 0.54)
+  timecop (~> 0.9)
+
+BUNDLED WITH
+   1.16.1

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2010-2018 Google, Inc. http://angularjs.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,68 @@
+# AtomicSidekiq
+AtomicSidekiq implements a reliable way of processing jobs using Sidekiq. By default, Sidekiq will retrieve jobs from the queue by removing it from Redis. If the job fails to complete (e.g. the process terminates unexpectdly mid-job), the job will be lost forever. This can be acceptable in many applications, but some application require higher levels of reliability, hence AtomicSidekiq will not erase any job from Redis until it's acknowledged that they have finished - otherwise, they are re-scheduled.
+
+The algorithm used by AtomicSidekiq supports both queue prioritization mechanisms: strict priority and weighted random.
+
+## Requirements
+AtomicSidekiq supports only Sidekiq 5+.
+
+## Installation
+```
+gem install atomic-sidekiq
+```
+
+Add to your server configuration (or create a new one if you don't have):
+```ruby
+Sidekiq.configure_server do |config|
+  config.atomic_fetch!
+end
+```
+
+## Configuration
+By default, jobs will expire and be re-queued after 1 hour if not acknowledged, and the "Collector" will check if for expired jobs every 60 seconds. This can be reconfigured as desired: _(Note that collection adds some overhead)_
+```ruby
+Sidekiq.configure_server do |config|
+  config.atomic_fetch!({
+    collection_interval: 5, # Unit: seconds
+    expiration_time: 1800   # Unit: seconds (30 minutes)
+  })
+end
+```
+
+## Benchmark
+### Reliability
+This benchmark tests Sidekiq's ability to recover from unexpected failures. The test script forces a failure randomly 1% of the time it's running a job and measures how many jobs are able to be completed:
+
+| Version       | Queued  | Processed | Lost  |
+|---------------|---------|-----------|-------|
+| Sidekiq       | 10,000  | 1,838     | 8,162 |
+| AtomicSidekiq | 10,000  | 10,000    | 0     |
+
+Since jobs run in parallel, when the process crashes it loses all jobs that had been retrieved and were running at the moment. AtomicSidekiq manage to retrieve all jobs and finish the work. The reliability script can be found at `./bin/sidekiqfail`. It terminates and restores Sidekiq several times until all jobs are processed or a maximum number of tries is reached.
+
+The test script can be run with the flag `-a` to use the **AtomicSidekiq::AtomicFetch** and without any flags to run with the default Sidekiq fetcher.
+
+_(Note: Sidekiq PRO comes with its own reliable fetcher, no benchmarks were run against that version. Only the free version has been tested)_
+
+### Performance
+The performance test uses the default settings for both fetchers (default and AtomicFetch) and times how long Sidekiq takes to process two loads, one of 10,000 and another one of 100,000 jobs.
+
+| Version       | Time Ellapsed (10k) | Throughput (10k) | Time Ellapsed (100k) | Throughput (100k) |
+|---------------|---------------------|------------------|----------------------|-------------------|
+| Sidekiq       | 6s                  | 166 jobs/sec     | 30s                  | 3,333 jobs/sec    |
+| AtomicSidekiq | 8s                  | 125 jobs/sec     | 1m10s                | 1,429 jobs/sec    |
+
+The reliability improvements of AtomicSidekiq come at the cost of less throughput. AtomicSidekiq's algorithm is linear instead of constant like Sidekiq's default, meaning that the cost of performance increases linearly as more jobs are added to the queue.
+
+## Tests
+```sh
+bundle exec rspec
+```
+
+You may also run the tests with Docker using `docker-compose` (it will automatially start a Redis server for the integration tests):
+```sh
+docker-compose run test
+```
+
+## Caveat
+This ensures that your job will be run completely **at least once**. It may run more than once if your job fails to acknowledge (e.g. the process terminates after performing a job but right before the ack is sent). _Note: This is better than the default Sidekiq though, which cannot give any guarantees on the number of times a job will be run._

data/atomic-sidekiq.gemspec

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+  s.name          = "atomic-sidekiq"
+  s.version       = "1.0.0"
+  s.date          = "2018-04-01"
+  s.summary       = "Reliable fetcher for Sidekiq"
+  s.description   = "Reliable fetcher for Sidekiq"
+  s.homepage      = "https://github.com/Colex/atomic-sidekiq"
+  s.authors       = ["Alex Correia Santos"]
+  s.email         = ["alex.santios@visiblealpha.com"]
+  s.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.require_paths = ["lib"]
+  s.license       = "MIT"
+
+  s.required_ruby_version = ">= 2.3"
+
+  s.add_development_dependency "bundler", "~> 1.12"
+  s.add_development_dependency "rake", "~> 11.3"
+  s.add_development_dependency "rspec", "~> 3.6"
+  s.add_development_dependency "rubocop", "~> 0.54"
+  s.add_development_dependency "byebug", "~> 10.0"
+  s.add_development_dependency "timecop", "~> 0.9"
+
+  s.add_runtime_dependency "sidekiq", "~> 5.0"
+end

data/bin/release

@@ -0,0 +1,9 @@
+#!/bin/bash
+cp atomic-sidekiq.gemspec atomic-sidekiq.gemspec.backup
+sed -i.bak s/0.0.0/$1/g atomic-sidekiq.gemspec
+sed -i.bak s/2001-01-01/`date +%Y-%m-%d`/g atomic-sidekiq.gemspec
+gem build atomic-sidekiq.gemspec
+mv atomic-sidekiq.gemspec.backup atomic-sidekiq.gemspec
+gem push atomic-sidekiq-$1.gem
+git tag -a v$1 -m "Release v$1"
+git push origin --tags

data/bin/sidekiqfail

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'sidekiq'
+
+max_tries = 10
+cutoff_tries = 100
+
+options = {
+  atomic_fetch: false
+}
+OptionParser.new do |opts|
+  opts.banner = "Usage: sidekiqfail [options]"
+
+  opts.on("-a", "--atomic-fetch", "Run the Sidekiq worker with the atomic-fetch fetcher") do |v|
+    options[:atomic_fetch] = v
+  end
+end.parse!
+
+Sidekiq.configure_client do |config|
+  config.redis = { db: 13 }
+end
+
+def processed
+  Sidekiq.redis { |conn| conn.get('done') }.to_i
+end
+
+def total
+  Sidekiq.redis { |conn| conn.get('total_enqueued') }.to_i
+end
+
+def inflight
+  counter = it = 0
+  loop do
+    it, keys = Sidekiq.redis { |c| c.scan(it, match: 'flight:*') }
+    counter += keys.count
+    it = it.to_i
+    break if it == 0
+  end
+  counter
+end
+
+def pending
+  total - processed
+end
+
+def print_report
+  Sidekiq.logger.error "Queued: #{total}"
+  Sidekiq.logger.error "Processed: #{processed}"
+  Sidekiq.logger.error "Lost: #{pending}"
+end
+
+args = ['-f', 'true', '-b', '10', '-j', '1000', '-t', '0.01']
+args.push('-a') if options[:atomic_fetch]
+pid = Process.fork { Process.exec('./bin/sidekiqload', *args) }
+loop do
+  Process.wait(pid)
+  puts "Processed before failure #{processed} out of #{total}"
+  break if pending == 0 || max_tries == 0 || cutoff_tries == 0
+  max_tries -= 1 if inflight == 0
+  cutoff_tries -= 1
+  args = ['-f', 'false', '-b', '0', '-j', '0', '-t', '0.01']
+  args.push('-a') if options[:atomic_fetch]
+  pid = Process.fork { Process.exec('./bin/sidekiqload', *args) }
+end
+print_report

data/bin/sidekiqload

@@ -0,0 +1,227 @@
+#!/usr/bin/env ruby
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+
+#require 'ruby-prof'
+Bundler.require(:default)
+
+require 'sidekiq/cli'
+require 'sidekiq/launcher'
+require 'optparse'
+require_relative '../lib/atomic-sidekiq'
+
+$options = {
+  batches: 10,
+  jobs: 1_000,
+  terminate: 0,
+  flush: true,
+  expiration: 2,
+  atomic_fetch: false
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: sidekiqload [options]"
+
+  opts.on("-b n", "--batches=n", "Number of job batches to be created [default 10]") do |v|
+    $options[:batches] = v.to_i
+  end
+  opts.on("-j n", "--jobs=n", "Number of jobs in each batch [default 1000]") do |v|
+    $options[:jobs] = v.to_i
+  end
+  opts.on("-t n", "--terminate=n", "Probability of terminating the thread [default 0]") do |v|
+    $options[:terminate] = v.to_f
+  end
+  opts.on("-f n", "--flush=n", "Flush all jobs that have been created before [default true]") do |v|
+    $options[:flush] = (v == 'true')
+  end
+  opts.on("-e n", "--expiration=n", "Expiration time in seconds for a job [default 2]") do |v|
+    $options[:expiration] = v.to_i
+  end
+  opts.on("-a", "--atomic-fetch", "Run the Sidekiq worker with the atomic-fetch fetcher") do |v|
+    $options[:atomic_fetch] = v
+  end
+end.parse!
+
+include Sidekiq::Util
+
+Sidekiq.configure_server do |config|
+  #config.options[:concurrency] = 1
+  config.redis = { db: 13 }
+  config.options[:queues] << 'default'
+  config.logger.level = Logger::ERROR
+  config.average_scheduled_poll_interval = 2
+  config.atomic_fetch!({
+    collection_interval: 10,
+    expiration_time: $options[:expiration],
+  }) if $options[:atomic_fetch]
+end
+
+class LoadWorker
+  include Sidekiq::Worker
+  sidekiq_options retry: 1
+  sidekiq_retry_in do |x|
+    1
+  end
+
+  def perform(batch, idx)
+    if rand < $options[:terminate]
+      Sidekiq.logger.error("Terminating on job #{idx}")
+      Process.kill("KILL", Process.pid)
+    end
+    begin
+      Sidekiq.redis do |conn|
+        conn.eval("""
+          local lock = redis.call('get', 'jobs:#{batch}:#{idx}')
+          if lock then return nil end
+          redis.call('set', 'jobs:#{batch}:#{idx}', '1')
+          redis.call('incr', 'done')
+          return nil
+        """)
+      end
+    rescue e
+      Sidekiq.logger.error(e)
+    end
+    #raise idx.to_s if idx % 100 == 1
+  end
+end
+
+# brew tap shopify/shopify
+# brew install toxiproxy
+# gem install toxiproxy
+#require 'toxiproxy'
+# simulate a non-localhost network for realer-world conditions.
+# adding 1ms of network latency has an ENORMOUS impact on benchmarks
+#Toxiproxy.populate([{
+    #"name": "redis",
+    #"listen": "127.0.0.1:6380",
+    #"upstream": "127.0.0.1:6379"
+#}])
+
+self_read, self_write = IO.pipe
+%w(INT TERM TSTP TTIN).each do |sig|
+  begin
+    trap sig do
+      puts("Killed with #{sig}")
+      self_write.puts(sig)
+    end
+  rescue ArgumentError
+    puts "Signal #{sig} not supported"
+  end
+end
+
+if ($options[:flush])
+  puts "Flushing database..."
+  Sidekiq.redis {|c| c.flushdb }
+end
+
+def handle_signal(launcher, sig)
+  Sidekiq.logger.debug "Got #{sig} signal"
+  case sig
+  when 'INT'
+    # Handle Ctrl-C in JRuby like MRI
+    # http://jira.codehaus.org/browse/JRUBY-4637
+    raise Interrupt
+  when 'TERM'
+    # Heroku sends TERM and then waits 10 seconds for process to exit.
+    raise Interrupt
+  when 'TSTP'
+    Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+    launcher.quiet
+  when 'TTIN'
+    Thread.list.each do |thread|
+      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
+      if thread.backtrace
+        Sidekiq.logger.warn thread.backtrace.join("\n")
+      else
+        Sidekiq.logger.warn "<no backtrace available>"
+      end
+    end
+  end
+end
+
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+iter = $options[:batches]
+count = $options[:jobs]
+
+iter.times do |batch|
+  arr = Array.new(count) do
+    []
+  end
+  count.times do |idx|
+    arr[idx][0] = idx
+    arr[idx][1] = batch
+  end
+  Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
+end
+total_enqueued = Sidekiq.redis { |conn| conn.get('total_enqueued') }.to_i + count * iter
+Sidekiq.redis { |conn| conn.set('total_enqueued', total_enqueued) }
+Sidekiq.logger.error "Created #{count*iter} jobs (total: #{total_enqueued})"
+
+Monitoring = Thread.new do
+  def total
+    qsize, retries = Sidekiq.redis do |conn|
+      conn.pipelined do
+        conn.llen "queue:default"
+        conn.zcard "retry"
+      end
+    end.map(&:to_i)
+    qsize + retries
+  end
+
+  def inflight
+    counter = it = 0
+    loop do
+      it, keys = Sidekiq.redis { |c| c.scan(it, match: 'flight:*') }
+      counter += keys.count
+      it = it.to_i
+      break if it == 0
+    end
+    counter
+  end
+
+  watchdog("monitor thread") do
+    while true
+      sleep 1
+      #GC.start
+      _total = total
+      _inflight = inflight
+      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{_total} Inflight: #{_inflight}")
+      if _total == 0 && _inflight == 0
+        Sidekiq.logger.error("Done")
+        exit(0)
+      end
+    end
+  end
+end
+
+begin
+  # RubyProf::exclude_threads = [ Monitoring ]
+  #RubyProf.start
+  fire_event(:startup)
+  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+    launcher = Sidekiq::Launcher.new(Sidekiq.options)
+    launcher.run
+
+    while readable_io = IO.select([self_read])
+      signal = readable_io.first[0].gets.strip
+      handle_signal(launcher, signal)
+    end
+  #end
+rescue SystemExit => e
+  # Sidekiq.logger.error("Profiling...")
+  #result = RubyProf.stop
+  #printer = RubyProf::GraphHtmlPrinter.new(result)
+  #printer.print(File.new("output.html", "w"), :min_percent => 1)
+  # normal
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/bin/test

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+bundle check || bundle install
+bundle exec rspec

data/docker-compose.yml

@@ -0,0 +1,31 @@
+version: '2'
+
+services:
+  test:
+    build: .
+    command: bin/test
+    env_file:
+      - docker.env
+    depends_on:
+      - redis
+    volumes:
+      - .:/sidekiq-atomic
+    image: sidekiq-atomic
+    volumes_from:
+      - bundle
+
+  redis:
+    image: redis:3.2-alpine
+    ports:
+      - 6379:6379
+    volumes:
+      - redis:/var/lib/redis/data
+
+  bundle:
+    image: tianon/true
+    volumes:
+      - bundle:/bundle
+
+volumes:
+  redis:
+  bundle:

data/docker.env

@@ -0,0 +1 @@
+REDIS_URL=redis://redis:6379

data/lib/atomic-sidekiq.rb

@@ -0,0 +1 @@
+require_relative './atomic_sidekiq'

data/lib/atomic_sidekiq/atomic_fetch.rb

@@ -0,0 +1,55 @@
+module AtomicSidekiq
+  class AtomicFetch
+    IN_FLIGHT_KEY_PREFIX = "flight:"
+    DEFAULT_POLL_INTERVAL = 10 # seconds
+    DEFAULT_EXPIRATION_TIME = 3600 # seconds
+    DEFAULT_COLLECTION_INTERVAL = 60 # seconds
+
+    def initialize(options)
+      @retrieve_op = AtomicOperation::Retrieve.new(in_flight_prefix: IN_FLIGHT_KEY_PREFIX)
+      @strictly_ordered_queues = !!options[:strict]
+
+      atomic_fetch_opts = options.fetch(:atomic_fetch, {})
+      @expiration_time = atomic_fetch_opts.fetch(:expiration_time, DEFAULT_EXPIRATION_TIME)
+      @collection_interval = atomic_fetch_opts.fetch(:collection_wait_time, DEFAULT_COLLECTION_INTERVAL)
+      @poll_interval = atomic_fetch_opts.fetch(:poll_interval, DEFAULT_POLL_INTERVAL)
+      @@next_collection ||= Time.now
+      set_queues(options)
+    end
+
+    def retrieve_work
+      collect_dead_jobs!
+      work = retrieve_op.perform(ordered_queues, expire_at)
+      return UnitOfWork.new(*work) if work
+      sleep(poll_interval)
+      nil
+    end
+
+    private
+
+    attr_reader :retrieve_op, :queues, :strictly_ordered_queues,
+    :collection_interval, :poll_interval, :expiration_time
+
+    def set_queues(options)
+      @queues ||= options[:queues].map { |q| "queue:#{q}" }
+    end
+
+    def ordered_queues
+      if strictly_ordered_queues
+        queues
+      else
+        queues.shuffle.uniq
+      end
+    end
+
+    def collect_dead_jobs!
+      return if @@next_collection > Time.now
+      @@next_collection = Time.now + collection_interval
+      DeadJobCollector.collect!(ordered_queues)
+    end
+
+    def expire_at
+      Time.now.utc.to_i + expiration_time
+    end
+  end
+end

data/lib/atomic_sidekiq/atomic_operation/acknowledge.rb

@@ -0,0 +1,11 @@
+module AtomicSidekiq
+  module AtomicOperation
+    class Acknowledge < Base
+      def perform(queue:, job:)
+        redis do |conn|
+          conn.del(in_flight_job_key(queue, job))
+        end
+      end
+    end
+  end
+end

data/lib/atomic_sidekiq/atomic_operation/base.rb

@@ -0,0 +1,22 @@
+module AtomicSidekiq
+  module AtomicOperation
+    class Base
+      def initialize(in_flight_prefix:)
+        @in_flight_prefix = in_flight_prefix
+      end
+
+      protected
+
+      attr_reader :in_flight_prefix
+
+      def redis(&block)
+        Sidekiq.redis { |conn| block.call(conn) }
+      end
+
+      def in_flight_job_key(queue, job)
+        jid = JSON.parse(job)['jid']
+        "#{in_flight_prefix}#{queue}:#{jid}"
+      end
+    end
+  end
+end

data/lib/atomic_sidekiq/atomic_operation/expire.rb

@@ -0,0 +1,19 @@
+module AtomicSidekiq
+  module AtomicOperation
+    class Expire < Base
+      def initialize
+        super(in_flight_prefix: nil)
+      end
+
+      def perform(queue, in_flight_key)
+        redis do |conn|
+          conn.eval(EXPIRE_SCRIPT, [queue, in_flight_key], [Time.now.utc.to_i])
+        end
+      end
+
+      private
+
+      EXPIRE_SCRIPT = File.read(File.join(File.dirname(__FILE__), './lua_scripts/expire.lua'))
+    end
+  end
+end

data/lib/atomic_sidekiq/atomic_operation/lua_scripts/expire.lua

@@ -0,0 +1,15 @@
+local queue = KEYS[1]
+local in_flight_key = KEYS[2]
+local now = tonumber(ARGV[1])
+
+local job = redis.call('get', in_flight_key)
+if (not job) then return nil end
+
+local expiration = tonumber(string.match(job, '"expire_at":([0-9]*)'))
+if expiration > now then return nil end
+job = string.gsub(job, ',?"expire_at":[0-9]*', '')
+
+redis.call('lpush', queue, job)
+redis.call('del', in_flight_key)
+
+return { queue, job }

data/lib/atomic_sidekiq/atomic_operation/lua_scripts/retrieve.lua

@@ -0,0 +1,11 @@
+local queue = KEYS[1]
+local flight = KEYS[2]
+local expire_at = tonumber(ARGV[1])
+
+local job = redis.call('lpop', queue)
+if (not job) then return nil end
+job = job:sub(1,-2)..',"expire_at":'..expire_at.."}"
+
+local flight_key = flight..queue..':'..string.match(job, '"jid":"([^"]*)"')
+redis.call('set', flight_key, job)
+return { queue, job }

data/lib/atomic_sidekiq/atomic_operation/requeue.rb

@@ -0,0 +1,20 @@
+module AtomicSidekiq
+  module AtomicOperation
+    class Requeue < Base
+      def perform(queue:, job:)
+        redis do |conn|
+          requeue(conn, queue: queue, job: job)
+        end
+      end
+
+      private
+
+      def requeue(conn, queue:, job:)
+        conn.multi do
+          conn.rpush(queue, job)
+          conn.del(in_flight_job_key(queue, job))
+        end
+      end
+    end
+  end
+end

data/lib/atomic_sidekiq/atomic_operation/retrieve.rb

@@ -0,0 +1,23 @@
+module AtomicSidekiq
+  module AtomicOperation
+    class Retrieve < Base
+      def perform(queues, expire_at)
+        queues.each do |queue|
+          res = retrieve_from_queue(queue, expire_at.to_i)
+          return res if res
+        end
+        nil
+      end
+
+      private
+
+      RETRIEVE_SCRIPT = File.read(File.join(File.dirname(__FILE__), './lua_scripts/retrieve.lua'))
+
+      def retrieve_from_queue(queue, expire_at)
+        redis do |conn|
+          conn.eval(RETRIEVE_SCRIPT, [queue, in_flight_prefix], [expire_at])
+        end
+      end
+    end
+  end
+end

data/lib/atomic_sidekiq/dead_job_collector.rb

@@ -0,0 +1,43 @@
+module AtomicSidekiq
+  class DeadJobCollector
+    class << self
+      def collect!(queues)
+        queues.each { |q| new(q).collect! }
+      end
+    end
+
+    def initialize(queue, in_flight_prefix: AtomicFetch::IN_FLIGHT_KEY_PREFIX)
+      @queue            = queue
+      @in_flight_prefix = in_flight_prefix
+      @expire_op        = AtomicOperation::Expire.new
+    end
+
+    def collect!
+      each_keys { |job_key| expire!(job_key) }
+    end
+
+    private
+
+    attr_reader :queue, :in_flight_prefix, :expire_op
+
+    def expire!(job_key)
+      expire_op.perform(queue, job_key)
+    end
+
+    def each_keys(&block)
+      it = 0
+      Sidekiq.redis do |conn|
+        loop do
+          it, job_keys = conn.scan(it, match: keys_prefix)
+          it = it.to_i
+          job_keys.each { |job_key| block.call(job_key) }
+          break if it == 0
+        end
+      end
+    end
+
+    def keys_prefix
+      "#{in_flight_prefix}#{queue}:*"
+    end
+  end
+end

data/lib/atomic_sidekiq/sidekiq.rb

@@ -0,0 +1,6 @@
+module Sidekiq
+  def self.atomic_fetch!(opts = {})
+    self.options[:fetch] = AtomicSidekiq::AtomicFetch
+    self.options[:atomic_fetch] = opts
+  end
+end

data/lib/atomic_sidekiq/unit_of_work.rb

@@ -0,0 +1,28 @@
+module AtomicSidekiq
+  class UnitOfWork
+    attr_reader :queue, :job
+
+    def initialize(queue = nil, job = nil, in_flight_prefix: AtomicFetch::IN_FLIGHT_KEY_PREFIX)
+      @queue          = queue
+      @job            = job
+      @acknowledge_op = AtomicOperation::Acknowledge.new(in_flight_prefix: in_flight_prefix)
+      @requeue_op     = AtomicOperation::Requeue.new(in_flight_prefix: in_flight_prefix)
+    end
+
+    def acknowledge
+      acknowledge_op.perform(queue: queue, job: job)
+    end
+
+    def queue_name
+      "queue:#{queue.sub(/.*queue:/, '')}"
+    end
+
+    def requeue
+      requeue_op.perform(queue: queue, job: job)
+    end
+
+    private
+
+    attr_reader :acknowledge_op, :requeue_op
+  end
+end

data/lib/atomic_sidekiq.rb

@@ -0,0 +1,13 @@
+require 'sidekiq'
+require_relative 'atomic_sidekiq/sidekiq'
+require_relative 'atomic_sidekiq/unit_of_work'
+require_relative 'atomic_sidekiq/atomic_fetch'
+require_relative 'atomic_sidekiq/dead_job_collector'
+require_relative 'atomic_sidekiq/atomic_operation/base'
+require_relative 'atomic_sidekiq/atomic_operation/acknowledge'
+require_relative 'atomic_sidekiq/atomic_operation/requeue'
+require_relative 'atomic_sidekiq/atomic_operation/retrieve'
+require_relative 'atomic_sidekiq/atomic_operation/expire'
+
+module AtomicSidekiq
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: atomic-sidekiq
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Alex Correia Santos
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.54'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.54'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Reliable fetcher for Sidekiq
+email:
+- alex.santios@visiblealpha.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- atomic-sidekiq.gemspec
+- bin/release
+- bin/sidekiqfail
+- bin/sidekiqload
+- bin/test
+- docker-compose.yml
+- docker.env
+- lib/atomic-sidekiq.rb
+- lib/atomic_sidekiq.rb
+- lib/atomic_sidekiq/atomic_fetch.rb
+- lib/atomic_sidekiq/atomic_operation/acknowledge.rb
+- lib/atomic_sidekiq/atomic_operation/base.rb
+- lib/atomic_sidekiq/atomic_operation/expire.rb
+- lib/atomic_sidekiq/atomic_operation/lua_scripts/expire.lua
+- lib/atomic_sidekiq/atomic_operation/lua_scripts/retrieve.lua
+- lib/atomic_sidekiq/atomic_operation/requeue.rb
+- lib/atomic_sidekiq/atomic_operation/retrieve.rb
+- lib/atomic_sidekiq/dead_job_collector.rb
+- lib/atomic_sidekiq/sidekiq.rb
+- lib/atomic_sidekiq/unit_of_work.rb
+homepage: https://github.com/Colex/atomic-sidekiq
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Reliable fetcher for Sidekiq
+test_files: []