gitlab-sidekiq-fetcher 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f10159f679879ed622c3bbd336bb04d0da0c54d536b7270c8c81dcc1243cedfc
+  data.tar.gz: cbca779aacb710b4f8de222799ca88860f3745666d22b51ea3d7da7f5677f8a0
+SHA512:
+  metadata.gz: dd8376c3c379c325db87519e43f3ea36f96063b03b8884dc45363ff949cf3cda5341b20641f27dbb90be8a2188c0588ec92eb048b4055702aac6f7419f595a49
+  data.tar.gz: 02b0b4746cbe42960b89fc3b1b7f861f87b45317f6ae1f1f76d737333f80ac67c1553f6a8fdd3d44cacda87c08de88f9dee9b227a6d4b021d8bd58cacaa7fdee

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+coverage

data/.gitlab-ci.yml

@@ -0,0 +1,53 @@
+image: "ruby:2.5"
+
+before_script:
+  - ruby -v
+  - which ruby
+  - gem install bundler --no-ri --no-rdoc
+  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+
+variables:
+  REDIS_URL: "redis://redis"
+
+rspec:
+  stage: test
+  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
+  script:
+    - bundle exec rspec
+  services:
+    - redis:alpine
+  artifacts:
+    expire_in: 31d
+    when: always
+    paths:
+      - coverage/
+
+.integration:
+  stage: test
+  script:
+    - cd test
+    - bundle exec ruby reliability_test.rb
+  services:
+    - redis:alpine
+
+integration_semi:
+  extends: .integration
+  variables:
+    JOB_FETCHER: semi
+
+integration_reliable:
+  extends: .integration
+  variables:
+    JOB_FETCHER: reliable
+
+
+integration_basic:
+  extends: .integration
+  allow_failure: yes
+  variables:
+    JOB_FETCHER: basic
+
+
+# rubocop:
+#   script:
+#     - bundle exec rubocop

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+group :test do
+  gem "rspec", '~> 3'
+  gem "pry"
+  gem "sidekiq", '~> 5.0'
+  gem 'simplecov', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,50 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    connection_pool (2.2.2)
+    diff-lcs (1.3)
+    docile (1.3.1)
+    json (2.1.0)
+    method_source (0.9.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rack (2.0.5)
+    rack-protection (2.0.4)
+      rack
+    redis (4.0.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    sidekiq (5.2.2)
+      connection_pool (~> 2.2, >= 2.2.2)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.5, < 5)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry
+  rspec (~> 3)
+  sidekiq (~> 5.0)
+  simplecov
+
+BUNDLED WITH
+   1.17.1

data/LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,46 @@
+gitlab-sidekiq-fetcher
+======================
+
+`gitlab-sidekiq-fetcher` is an extension to Sidekiq that adds support for reliable
+fetches from Redis.
+
+It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
+
+There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
+semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
+However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.
+
+
+## Installation
+
+Add the following to your `Gemfile`:
+
+```ruby
+gem 'gitlab-sidekiq-fetcher', require: 'sidekiq-reliable-fetch'
+```
+
+## Configuration
+
+Enable reliable fetches by calling this gem from your Sidekiq configuration:
+
+```ruby
+Sidekiq.configure_server do |config|
+  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+
+  # …
+end
+```
+
+There is an additional parameter `config.options[:semi_reliable_fetch]` you can use to switch between two strategies:
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.options[:semi_reliable_fetch] = true # Default value is false
+
+  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+end
+```
+
+## License
+
+LGPL-3.0, see the LICENSE file.

data/RELEASE-GITLAB.md

@@ -0,0 +1,9 @@
+gitlab-sidekiq-fetcher
+======================
+
+# How to publish a new release?
+
+1. Dev-commit cycle
+2. Update the version in the gemspec file, commit and tag
+3. Build the gem: `gem build gitlab-sidekiq-fetcher.gemspec`
+4. Upload the gem: `gem push gitlab-sidekiq-fetcher-X.X.X.gem`

data/gitlab-sidekiq-fetcher.gemspec

@@ -0,0 +1,14 @@
+Gem::Specification.new do |s|
+  s.name        = 'gitlab-sidekiq-fetcher'
+  s.version     = '0.1.0'
+  s.authors     = ['TEA', 'GitLab']
+  s.email       = 'valery@gitlab.com'
+  s.license     = 'LGPL-3.0'
+  s.homepage    = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
+  s.summary     = 'Reliable fetch extension for Sidekiq'
+  s.description = 'Redis reliable queue pattern implemented in Sidekiq'
+  s.require_paths = ['lib']
+  s.files = `git ls-files`.split($\)
+  s.test_files  = []
+  s.add_dependency 'sidekiq', '~> 5'
+end

data/lib/sidekiq-reliable-fetch.rb

@@ -0,0 +1,5 @@
+require 'sidekiq'
+
+require_relative 'sidekiq/base_reliable_fetch'
+require_relative 'sidekiq/reliable_fetch'
+require_relative 'sidekiq/semi_reliable_fetch'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class BaseReliableFetch
+    DEFAULT_CLEANUP_INTERVAL = 60 * 60  # 1 hour
+    HEARTBEAT_INTERVAL       = 20       # seconds
+    HEARTBEAT_LIFESPAN       = 60       # seconds
+    HEARTBEAT_RETRY_DELAY    = 1        # seconds
+    WORKING_QUEUE_PREFIX     = 'working'
+
+    # Defines how often we try to take a lease to not flood our
+    # Redis server with SET requests
+    DEFAULT_LEASE_INTERVAL = 2 * 60 # seconds
+    LEASE_KEY              = 'reliable-fetcher-cleanup-lock'
+
+    # Defines the COUNT parameter that will be passed to Redis SCAN command
+    SCAN_COUNT = 1000
+
+    UnitOfWork = Struct.new(:queue, :job) do
+      def acknowledge
+        Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
+      end
+
+      def queue_name
+        queue.sub(/.*queue:/, '')
+      end
+
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.multi do |multi|
+            multi.lpush(queue, job)
+            multi.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job)
+          end
+        end
+      end
+    end
+
+    def self.setup_reliable_fetch!(config)
+      config.options[:fetch] = if config.options[:semi_reliable_fetch]
+                                 Sidekiq::SemiReliableFetch
+                               else
+                                 Sidekiq::ReliableFetch
+                               end
+
+      Sidekiq.logger.info('GitLab reliable fetch activated!')
+
+      start_heartbeat_thread
+    end
+
+    def self.start_heartbeat_thread
+      Thread.new do
+        loop do
+          begin
+            heartbeat
+
+            sleep HEARTBEAT_INTERVAL
+          rescue => e
+            Sidekiq.logger.error("Heartbeat thread error: #{e.message}")
+
+            sleep HEARTBEAT_RETRY_DELAY
+          end
+        end
+      end
+    end
+
+    def self.pid
+      @pid ||= ::Process.pid
+    end
+
+    def self.hostname
+      @hostname ||= Socket.gethostname
+    end
+
+    def self.heartbeat
+      Sidekiq.redis do |conn|
+        conn.set(heartbeat_key(hostname, pid), 1, ex: HEARTBEAT_LIFESPAN)
+      end
+
+      Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
+    end
+
+    def self.bulk_requeue(inprogress, _options)
+      return if inprogress.empty?
+
+      Sidekiq.logger.debug('Re-queueing terminated jobs')
+
+      Sidekiq.redis do |conn|
+        inprogress.each do |unit_of_work|
+          conn.multi do |multi|
+            multi.lpush(unit_of_work.queue, unit_of_work.job)
+            multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
+          end
+        end
+      end
+
+      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+    rescue => e
+      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
+    end
+
+    def self.heartbeat_key(hostname, pid)
+      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
+    end
+
+    def self.working_queue_name(queue)
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+    end
+
+    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
+                :queues, :use_semi_reliable_fetch,
+                :strictly_ordered_queues
+
+    def initialize(options)
+      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
+      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
+      @last_try_to_take_lease_at = 0
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues].map { |q| "queue:#{q}" }
+    end
+
+    def retrieve_work
+      clean_working_queues! if take_lease
+
+      retrieve_unit_of_work
+    end
+
+    def retrieve_unit_of_work
+      raise NotImplementedError,
+        "#{self.class} does not implement #{__method__}"
+    end
+
+    private
+
+    def clean_working_queue!(working_queue)
+      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
+
+      Sidekiq.redis do |conn|
+        count = 0
+
+        while conn.rpoplpush(working_queue, original_queue) do
+          count += 1
+        end
+
+        Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
+      end
+    end
+
+    # Detect "old" jobs and requeue them because the worker they were assigned
+    # to probably failed miserably.
+    def clean_working_queues!
+      Sidekiq.logger.info("Cleaning working queues")
+
+      Sidekiq.redis do |conn|
+        conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
+          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
+          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
+
+          continue if hostname.nil? || pid.nil?
+
+          clean_working_queue!(key) if worker_dead?(hostname, pid)
+        end
+      end
+    end
+
+    def worker_dead?(hostname, pid)
+      Sidekiq.redis do |conn|
+        !conn.get(self.class.heartbeat_key(hostname, pid))
+      end
+    end
+
+    def take_lease
+      return unless allowed_to_take_a_lease?
+
+      @last_try_to_take_lease_at = Time.now.to_f
+
+      Sidekiq.redis do |conn|
+        conn.set(LEASE_KEY, 1, nx: true, ex: cleanup_interval)
+      end
+    end
+
+    def allowed_to_take_a_lease?
+      Time.now.to_f - last_try_to_take_lease_at > lease_interval
+    end
+  end
+end

data/lib/sidekiq/reliable_fetch.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class ReliableFetch < BaseReliableFetch
+    # For reliable fetch we don't use Redis' blocking operations so
+    # we inject a regular sleep into the loop.
+    RELIABLE_FETCH_IDLE_TIMEOUT = 5 # seconds
+
+    attr_reader :queues_iterator, :queues_size
+
+    def initialize(options)
+      super
+
+      @queues_size = queues.size
+      @queues_iterator = queues.cycle
+    end
+
+    private
+
+    def retrieve_unit_of_work
+      @queues_iterator.rewind if strictly_ordered_queues
+
+      queues_size.times do
+        queue = queues_iterator.next
+
+        work = Sidekiq.redis do |conn|
+          conn.rpoplpush(queue, self.class.working_queue_name(queue))
+        end
+
+        return UnitOfWork.new(queue, work) if work
+      end
+
+      # We didn't find a job in any of the configured queues. Let's sleep a bit
+      # to avoid uselessly burning too much CPU
+      sleep(RELIABLE_FETCH_IDLE_TIMEOUT)
+
+      nil
+    end
+  end
+end

data/lib/sidekiq/semi_reliable_fetch.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class SemiReliableFetch < BaseReliableFetch
+    # We want the fetch operation to timeout every few seconds so the thread
+    # can check if the process is shutting down. This constant is only used
+    # for semi-reliable fetch.
+    SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
+
+    def initialize(options)
+      super
+
+      if strictly_ordered_queues
+        @queues = @queues.uniq
+        @queues << SEMI_RELIABLE_FETCH_TIMEOUT
+      end
+    end
+
+    private
+
+    def retrieve_unit_of_work
+      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
+      return unless work
+
+      unit_of_work = UnitOfWork.new(*work)
+
+      Sidekiq.redis do |conn|
+        conn.lpush(self.class.working_queue_name(unit_of_work.queue), unit_of_work.job)
+      end
+
+      unit_of_work
+    end
+
+    def queues_cmd
+      if strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle.uniq
+        queues << SEMI_RELIABLE_FETCH_TIMEOUT
+        queues
+      end
+    end
+  end
+end

data/spec/base_reliable_fetch_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require 'fetch_shared_examples'
+require 'sidekiq/base_reliable_fetch'
+require 'sidekiq/reliable_fetch'
+require 'sidekiq/semi_reliable_fetch'
+
+describe Sidekiq::BaseReliableFetch do
+  before { Sidekiq.redis(&:flushdb) }
+
+  describe 'UnitOfWork' do
+    let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
+
+    describe '#requeue' do
+      it 'requeues job' do
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+
+        uow = fetcher.retrieve_work
+
+        uow.requeue
+
+        expect(Sidekiq::Queue.new('foo').size).to eq 1
+        expect(working_queue_size('foo')).to eq 0
+      end
+    end
+
+    describe '#acknowledge' do
+      it 'acknowledges job' do
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+
+        uow = fetcher.retrieve_work
+
+        expect { uow.acknowledge }
+          .to change { working_queue_size('foo') }.by(-1)
+
+        expect(Sidekiq::Queue.new('foo').size).to eq 0
+      end
+    end
+  end
+
+  describe '.bulk_requeue' do
+    it 'requeues the bulk' do
+      queue1 = Sidekiq::Queue.new('foo')
+      queue2 = Sidekiq::Queue.new('bar')
+
+      expect(queue1.size).to eq 0
+      expect(queue2.size).to eq 0
+
+      uow = described_class::UnitOfWork
+      jobs = [ uow.new('queue:foo', 'bob'), uow.new('queue:foo', 'bar'), uow.new('queue:bar', 'widget') ]
+      described_class.bulk_requeue(jobs, queues: [])
+
+      expect(queue1.size).to eq 2
+      expect(queue2.size).to eq 1
+    end
+  end
+
+  it 'sets heartbeat' do
+    config = double(:sidekiq_config, options: {})
+
+    heartbeat_thread = described_class.setup_reliable_fetch!(config)
+
+    Sidekiq.redis do |conn|
+      sleep 0.2 # Give the time to heartbeat thread to make a loop
+
+      heartbeat_key = described_class.heartbeat_key(Socket.gethostname, ::Process.pid)
+      heartbeat = conn.get(heartbeat_key)
+
+      expect(heartbeat).not_to be_nil
+    end
+
+    heartbeat_thread.kill
+  end
+end

data/spec/fetch_shared_examples.rb

@@ -0,0 +1,118 @@
+shared_examples 'a Sidekiq fetcher' do
+  let(:queues) { ['assigned'] }
+
+  before { Sidekiq.redis(&:flushdb) }
+
+  describe '#retrieve_work' do
+    let(:fetcher) { described_class.new(queues: ['assigned']) }
+
+    it 'retrieves the job and puts it to working queue' do
+      Sidekiq.redis { |conn| conn.rpush('queue:assigned', 'msg') }
+
+      uow = fetcher.retrieve_work
+
+      expect(working_queue_size('assigned')).to eq 1
+      expect(uow.queue_name).to eq 'assigned'
+      expect(uow.job).to eq 'msg'
+      expect(Sidekiq::Queue.new('assigned').size).to eq 0
+    end
+
+    it 'does not retrieve a job from foreign queue' do
+      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', 'msg') }
+
+      expect(fetcher.retrieve_work).to be_nil
+    end
+
+    it 'requeues jobs from dead working queue' do
+      Sidekiq.redis do |conn|
+        conn.rpush(other_process_working_queue_name('assigned'), 'msg')
+      end
+
+      uow = fetcher.retrieve_work
+
+      expect(uow.job).to eq 'msg'
+
+      Sidekiq.redis do |conn|
+        expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
+      end
+    end
+
+    it 'does not requeue jobs from live working queue' do
+      working_queue = live_other_process_working_queue_name('assigned')
+
+      Sidekiq.redis do |conn|
+        conn.rpush(working_queue, 'msg')
+      end
+
+      uow = fetcher.retrieve_work
+
+      expect(uow).to be_nil
+
+      Sidekiq.redis do |conn|
+        expect(conn.llen(working_queue)).to eq 1
+      end
+    end
+
+    it 'does not clean up orphaned jobs more than once per cleanup interval' do
+      Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
+
+      expect_any_instance_of(described_class)
+        .to receive(:clean_working_queues!).once
+
+      threads = 10.times.map do
+        Thread.new do
+          described_class.new(queues: ['assigned']).retrieve_work
+        end
+      end
+
+      threads.map(&:join)
+    end
+
+    it 'retrieves by order when strictly order is enabled' do
+      fetcher = described_class.new(strict: true, queues: ['first', 'second'])
+
+      Sidekiq.redis do |conn|
+        conn.rpush('queue:first', ['msg3', 'msg2', 'msg1'])
+        conn.rpush('queue:second', 'msg4')
+      end
+
+      jobs = (1..4).map { fetcher.retrieve_work.job }
+
+      expect(jobs).to eq ['msg1', 'msg2', 'msg3', 'msg4']
+    end
+
+    it 'does not starve any queue when queues are not strictly ordered' do
+      fetcher = described_class.new(queues: ['first', 'second'])
+
+      Sidekiq.redis do |conn|
+        conn.rpush('queue:first', (1..200).map { |i| "msg#{i}" })
+        conn.rpush('queue:second', 'this_job_should_not_stuck')
+      end
+
+      jobs = (1..100).map { fetcher.retrieve_work.job }
+
+      expect(jobs).to include 'this_job_should_not_stuck'
+    end
+  end
+end
+
+def working_queue_size(queue_name)
+  Sidekiq.redis do |c|
+    c.llen(Sidekiq::BaseReliableFetch.working_queue_name("queue:#{queue_name}"))
+  end
+end
+
+def other_process_working_queue_name(queue)
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
+end
+
+def live_other_process_working_queue_name(queue)
+  pid = ::Process.pid + 1
+  hostname = Socket.gethostname
+
+  Sidekiq.redis do |conn|
+    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key(hostname, pid), 1)
+  end
+
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}"
+end

data/spec/reliable_fetch_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+require 'fetch_shared_examples'
+require 'sidekiq/reliable_fetch'
+
+describe Sidekiq::ReliableFetch do
+  include_examples 'a Sidekiq fetcher'
+end

data/spec/semi_reliable_fetch_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+require 'fetch_shared_examples'
+require 'sidekiq/semi_reliable_fetch'
+
+describe Sidekiq::SemiReliableFetch do
+  include_examples 'a Sidekiq fetcher'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,115 @@
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/api'
+require 'pry'
+require 'simplecov'
+
+SimpleCov.start
+
+REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost:6379/10'
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: REDIS_URL }
+end
+
+Sidekiq.logger.level = Logger::ERROR
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

data/test/README.md

@@ -0,0 +1,34 @@
+# How to run
+
+```
+cd test
+bundle exec ruby reliability_test.rb
+```
+
+You can adjust some parameters of the test in the `config.rb`
+
+
+# How it works
+
+This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
+number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
+
+# How to run tests
+
+To run rspec:
+
+```
+bundle exec rspec
+```
+
+To run performance tests:
+
+```
+cd test
+JOB_FETCHER=semi bundle exec ruby reliability_test.rb
+```
+
+JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
+
+To run both kind of tests you need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
+`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).

data/test/config.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative '../lib/sidekiq/base_reliable_fetch'
+require_relative '../lib/sidekiq/reliable_fetch'
+require_relative '../lib/sidekiq/semi_reliable_fetch'
+require_relative 'worker'
+
+REDIS_FINISHED_LIST = 'reliable-fetcher-finished-jids'
+
+NUMBER_OF_WORKERS = ENV['NUMBER_OF_WORKERS'] || 10
+NUMBER_OF_JOBS = ENV['NUMBER_OF_JOBS'] || 1000
+JOB_FETCHER = (ENV['JOB_FETCHER'] || :semi).to_sym # :basic, :semi, :reliable
+TEST_CLEANUP_INTERVAL = 20
+TEST_LEASE_INTERVAL = 5
+WAIT_CLEANUP = TEST_CLEANUP_INTERVAL +
+               TEST_LEASE_INTERVAL +
+               Sidekiq::ReliableFetch::HEARTBEAT_LIFESPAN
+
+Sidekiq.configure_server do |config|
+  if %i[semi reliable].include?(JOB_FETCHER)
+    config.options[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
+
+    # We need to override these parameters to not wait too long
+    # The default values are good for production use only
+    # These will be ignored for :basic
+    config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+    config.options[:lease_interval] = TEST_LEASE_INTERVAL
+
+    Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+  end
+end

data/test/reliability_test.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/cli'
+require_relative 'config'
+
+def spawn_workers_and_stop_them_on_a_half_way
+  pids = spawn_workers
+
+  wait_until do |queue_size|
+    queue_size < NUMBER_OF_JOBS / 2
+  end
+
+  first_half_pids, second_half_pids = split_array(pids)
+
+  puts 'Killing half of the workers...'
+  signal_to_workers('KILL', first_half_pids)
+
+  puts 'Stopping another half of the workers...'
+  signal_to_workers('TERM', second_half_pids)
+end
+
+def spawn_workers_and_let_them_finish
+  puts 'Spawn workers and let them finish...'
+
+  pids = spawn_workers
+
+  wait_until do |queue_size|
+    queue_size.zero?
+  end
+
+  if %i[semi reliable].include? JOB_FETCHER
+    puts 'Waiting for clean up process that will requeue dead jobs...'
+    sleep WAIT_CLEANUP
+  end
+
+  signal_to_workers('TERM', pids)
+end
+
+def wait_until
+  loop do
+    sleep 3
+
+    queue_size = current_queue_size
+    puts "Jobs in the queue:#{queue_size}"
+
+    break if yield(queue_size)
+  end
+end
+
+def signal_to_workers(signal, pids)
+  pids.each { |pid| Process.kill(signal, pid) }
+  pids.each { |pid| Process.wait(pid) }
+end
+
+def spawn_workers
+  pids = []
+  NUMBER_OF_WORKERS.times do
+    pids << spawn('sidekiq -r ./config.rb')
+  end
+
+  pids
+end
+
+def current_queue_size
+  Sidekiq.redis { |c| c.llen('queue:default') }
+end
+
+def duplicates
+  Sidekiq.redis { |c| c.llen(REDIS_FINISHED_LIST) }
+end
+
+# Splits array into two halves
+def split_array(arr)
+  first_arr = arr.take(arr.size / 2)
+  second_arr = arr - first_arr
+  [first_arr, second_arr]
+end
+
+##########################################################
+
+puts '########################################'
+puts "Mode: #{JOB_FETCHER}"
+puts '########################################'
+
+Sidekiq.redis(&:flushdb)
+
+jobs = []
+
+NUMBER_OF_JOBS.times do
+  jobs << TestWorker.perform_async
+end
+
+puts "Queued #{NUMBER_OF_JOBS} jobs"
+
+spawn_workers_and_stop_them_on_a_half_way
+spawn_workers_and_let_them_finish
+
+jobs_lost = 0
+
+Sidekiq.redis do |redis|
+  jobs.each do |job|
+    next if redis.lrem(REDIS_FINISHED_LIST, 1, job) == 1
+    jobs_lost += 1
+  end
+end
+
+puts "Remaining unprocessed: #{jobs_lost}"
+puts "Duplicates found: #{duplicates}"
+
+if jobs_lost.zero? && duplicates.zero?
+  exit 0
+else
+  exit 1
+end

data/test/worker.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class TestWorker
+  include Sidekiq::Worker
+
+  def perform
+    # To mimic long running job and to increase the probability of losing the job
+    sleep 1
+
+    Sidekiq.redis do |redis|
+      redis.lpush(REDIS_FINISHED_LIST, get_sidekiq_job_id)
+    end
+  end
+
+  def get_sidekiq_job_id
+    context_data = Thread.current[:sidekiq_context]&.first
+
+    return unless context_data
+
+    index = context_data.index('JID-')
+
+    return unless index
+
+    context_data[index + 4..-1]
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: gitlab-sidekiq-fetcher
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- TEA
+- GitLab
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-12-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+description: Redis reliable queue pattern implemented in Sidekiq
+email: valery@gitlab.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".gitlab-ci.yml"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- RELEASE-GITLAB.md
+- gitlab-sidekiq-fetcher.gemspec
+- lib/sidekiq-reliable-fetch.rb
+- lib/sidekiq/base_reliable_fetch.rb
+- lib/sidekiq/reliable_fetch.rb
+- lib/sidekiq/semi_reliable_fetch.rb
+- spec/base_reliable_fetch_spec.rb
+- spec/fetch_shared_examples.rb
+- spec/reliable_fetch_spec.rb
+- spec/semi_reliable_fetch_spec.rb
+- spec/spec_helper.rb
+- test/README.md
+- test/config.rb
+- test/reliability_test.rb
+- test/worker.rb
+homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
+licenses:
+- LGPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Reliable fetch extension for Sidekiq
+test_files: []