gitlab-sidekiq-fetcher 0.5.5 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb384451139d1638854cd94a673b12e0f7fd5a42c663535da0145fd153ba4c51
-  data.tar.gz: 43851081b6406fe8a876824ef034c548665382707db0aaf41c84c186a98419b0
+  metadata.gz: c7be23d59956ffa44288a1c870bcca66fd0119682f810325d71a3ebaa8b76e80
+  data.tar.gz: 013a7124f61044572ad93335e95c18357c60804dd89024d987485b2d87775787
 SHA512:
-  metadata.gz: 617032e226ffc898b1c411b0f7598169272bc8521156b9eadcb35b48a139cd854f1b12a2fd98e7fd174128727f0b8c60917780cd2b6dec82cb1370a6260b20ee
-  data.tar.gz: 5e56e3840a5f0a4f437ec298de7b68ef796dcfa7d31e69117398ceb22b4ed07fbb4eac4e711b344f5134fd748f9afa476d0d4ffc216cc02cd4eec5b9db90a9c2
+  metadata.gz: 92653bc5f9b5729f4dd50a8243a869c20d9621a1a9d25c46d729e735895e0f2d4d940c5a766803f1a5fd908ab0d9340f27d255dbd99f31bab4923e2f539c1882
+  data.tar.gz: d763b8b0ee3c2522752130fac86b83e67e8513faf919dd361aad9896aac684809650959b1627dd38e0440343c063f95b807b67fc31a10217ce1f15c428759803

data/.gitlab-ci.yml

@@ -40,7 +40,6 @@ integration_reliable:
   variables:
     JOB_FETCHER: reliable
 
-
 integration_basic:
   extends: .integration
   allow_failure: yes
@@ -63,7 +62,6 @@ term_interruption:
   services:
     - redis:alpine
 
-
 # rubocop:
 #   script:
 #     - bundle exec rubocop

data/Gemfile

@@ -7,6 +7,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 group :test do
   gem "rspec", '~> 3'
   gem "pry"
-  gem "sidekiq", '~> 5.0'
+  gem "sidekiq", '~> 6.1'
   gem 'simplecov', require: false
 end

data/Gemfile.lock

@@ -2,7 +2,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     docile (1.3.1)
     json (2.1.0)
@@ -10,10 +10,8 @@ GEM
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.0.5)
-    rack-protection (2.0.4)
-      rack
-    redis (4.0.2)
+    rack (2.2.3)
+    redis (4.2.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -27,10 +25,10 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    sidekiq (5.2.2)
-      connection_pool (~> 2.2, >= 2.2.2)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
+    sidekiq (6.1.0)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -43,8 +41,8 @@ PLATFORMS
 DEPENDENCIES
   pry
   rspec (~> 3)
-  sidekiq (~> 5.0)
+  sidekiq (~> 6.1)
   simplecov
 
 BUNDLED WITH
-   1.17.1
+   1.17.2

data/README.md

@@ -6,6 +6,10 @@ fetches from Redis.
 
 It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
 
+**IMPORTANT NOTE:** Since version `0.7.0` this gem works only with `sidekiq >= 6.1` (which introduced Fetch API breaking changes). Please use version `~> 0.5` if you use older version of the `sidekiq` .
+
+**UPGRADE NOTE:** If upgrading from 0.7.0, strongly consider a full deployed step on 0.7.1 before 0.8.0; that fixes a bug in the queue name validation that will hit if sidekiq nodes running 0.7.0 see working queues named by 0.8.0.  See https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/-/merge_requests/22
+
 There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
 semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
 However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.5.5'
+  s.version       = '0.8.0'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'
@@ -10,5 +10,5 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.files         = `git ls-files`.split($\)
   s.test_files    = []
-  s.add_dependency 'sidekiq', '~> 5'
+  s.add_dependency 'sidekiq', '~> 6.1'
 end

data/lib/sidekiq/base_reliable_fetch.rb

@@ -45,11 +45,13 @@ module Sidekiq
     end
 
     def self.setup_reliable_fetch!(config)
-      config.options[:fetch] = if config.options[:semi_reliable_fetch]
-                                 Sidekiq::SemiReliableFetch
-                               else
-                                 Sidekiq::ReliableFetch
-                               end
+      fetch_strategy = if config.options[:semi_reliable_fetch]
+                         Sidekiq::SemiReliableFetch
+                       else
+                         Sidekiq::ReliableFetch
+                       end
+
+      config.options[:fetch] = fetch_strategy.new(config.options)
 
       Sidekiq.logger.info('GitLab reliable fetch activated!')
 
@@ -92,7 +94,44 @@ module Sidekiq
       Sidekiq.logger.debug("Heartbeat for #{identity}")
     end
 
-    def self.bulk_requeue(inprogress, _options)
+    def self.worker_dead?(identity, conn)
+      !conn.get(heartbeat_key(identity))
+    end
+
+    def self.heartbeat_key(identity)
+      "reliable-fetcher-heartbeat-#{identity.gsub(':', '-')}"
+    end
+
+    def self.working_queue_name(queue)
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{identity}"
+    end
+
+    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
+                :queues, :use_semi_reliable_fetch,
+                :strictly_ordered_queues
+
+    def initialize(options)
+      raise ArgumentError, 'missing queue list' unless options[:queues]
+
+      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
+      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
+      @last_try_to_take_lease_at = 0
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues].map { |q| "queue:#{q}" }
+    end
+
+    def retrieve_work
+      clean_working_queues! if take_lease
+
+      retrieve_unit_of_work
+    end
+
+    def retrieve_unit_of_work
+      raise NotImplementedError,
+            "#{self.class} does not implement #{__method__}"
+    end
+
+    def bulk_requeue(inprogress, _options)
       return if inprogress.empty?
 
       Sidekiq.redis do |conn|
@@ -100,7 +139,7 @@ module Sidekiq
           conn.multi do |multi|
             preprocess_interrupted_job(unit_of_work.job, unit_of_work.queue, multi)
 
-            multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
+            multi.lrem(self.class.working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
           end
         end
       end
@@ -108,15 +147,9 @@ module Sidekiq
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
     end
 
-    def self.clean_working_queue!(original_queue, working_queue)
-      Sidekiq.redis do |conn|
-        while job = conn.rpop(working_queue)
-          preprocess_interrupted_job(job, original_queue)
-        end
-      end
-    end
+    private
 
-    def self.preprocess_interrupted_job(job, queue, conn = nil)
+    def preprocess_interrupted_job(job, queue, conn = nil)
       msg = Sidekiq.load_json(job)
       msg['interrupted_count'] = msg['interrupted_count'].to_i + 1
 
@@ -127,7 +160,21 @@ module Sidekiq
       end
     end
 
-    def self.extract_queue_and_identity(key)
+    # If you want this method to be run in a scope of multi connection
+    # you need to pass it
+    def requeue_job(queue, msg, conn)
+      with_connection(conn) do |conn|
+        conn.lpush(queue, Sidekiq.dump_json(msg))
+      end
+
+      Sidekiq.logger.info(
+        message: "Pushed job #{msg['jid']} back to queue #{queue}",
+        jid: msg['jid'],
+        queue: queue
+      )
+    end
+
+    def extract_queue_and_identity(key)
       # New identity format is "{hostname}:{pid}:{randomhex}
       # Old identity format is "{hostname}:{pid}"
       # Queue names may also have colons (namespaced).
@@ -142,7 +189,7 @@ module Sidekiq
 
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
-    def self.clean_working_queues!
+    def clean_working_queues!
       Sidekiq.logger.info('Cleaning working queues')
 
       Sidekiq.redis do |conn|
@@ -151,30 +198,26 @@ module Sidekiq
 
           next if original_queue.nil? || identity.nil?
 
-          clean_working_queue!(original_queue, key) if worker_dead?(identity, conn)
+          clean_working_queue!(original_queue, key) if self.class.worker_dead?(identity, conn)
         end
       end
     end
 
-    def self.worker_dead?(identity, conn)
-      !conn.get(heartbeat_key(identity))
-    end
-
-    def self.heartbeat_key(identity)
-      "reliable-fetcher-heartbeat-#{identity.gsub(':', '-')}"
-    end
-
-    def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{identity}"
+    def clean_working_queue!(original_queue, working_queue)
+      Sidekiq.redis do |conn|
+        while job = conn.rpop(working_queue)
+          preprocess_interrupted_job(job, original_queue)
+        end
+      end
     end
 
-    def self.interruption_exhausted?(msg)
+    def interruption_exhausted?(msg)
       return false if max_retries_after_interruption(msg['class']) < 0
 
       msg['interrupted_count'].to_i >= max_retries_after_interruption(msg['class'])
     end
 
-    def self.max_retries_after_interruption(worker_class)
+    def max_retries_after_interruption(worker_class)
       max_retries_after_interruption = nil
 
       max_retries_after_interruption ||= begin
@@ -187,7 +230,7 @@ module Sidekiq
       max_retries_after_interruption
     end
 
-    def self.send_to_quarantine(msg, multi_connection = nil)
+    def send_to_quarantine(msg, multi_connection = nil)
       Sidekiq.logger.warn(
         class: msg['class'],
         jid: msg['jid'],
@@ -198,52 +241,13 @@ module Sidekiq
       Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
     end
 
-    # If you want this method to be run is a scope of multi connection
-    # you need to pass it
-    def self.requeue_job(queue, msg, conn)
-      with_connection(conn) do |conn|
-        conn.lpush(queue, Sidekiq.dump_json(msg))
-      end
-
-      Sidekiq.logger.info(
-        message: "Pushed job #{msg['jid']} back to queue #{queue}",
-        jid: msg['jid'],
-        queue: queue
-      )
-    end
-
     # Yield block with an existing connection or creates another one
-    def self.with_connection(conn, &block)
+    def with_connection(conn)
       return yield(conn) if conn
 
-      Sidekiq.redis { |conn| yield(conn) }
-    end
-
-    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
-                :queues, :use_semi_reliable_fetch,
-                :strictly_ordered_queues
-
-    def initialize(options)
-      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
-      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
-      @last_try_to_take_lease_at = 0
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
-    end
-
-    def retrieve_work
-      self.class.clean_working_queues! if take_lease
-
-      retrieve_unit_of_work
+      Sidekiq.redis { |redis_conn| yield(redis_conn) }
     end
 
-    def retrieve_unit_of_work
-      raise NotImplementedError,
-        "#{self.class} does not implement #{__method__}"
-    end
-
-    private
-
     def take_lease
       return unless allowed_to_take_a_lease?
 

data/lib/sidekiq/reliable_fetch.rb

@@ -6,23 +6,21 @@ module Sidekiq
     # we inject a regular sleep into the loop.
     RELIABLE_FETCH_IDLE_TIMEOUT = 5 # seconds
 
-    attr_reader :queues_iterator, :queues_size
+    attr_reader :queues_size
 
     def initialize(options)
       super
 
+      @queues = queues.uniq if strictly_ordered_queues
       @queues_size = queues.size
-      @queues_iterator = queues.cycle
     end
 
     private
 
     def retrieve_unit_of_work
-      @queues_iterator.rewind if strictly_ordered_queues
-
-      queues_size.times do
-        queue = queues_iterator.next
+      queues_list = strictly_ordered_queues ? queues : queues.shuffle
 
+      queues_list.each do |queue|
         work = Sidekiq.redis do |conn|
           conn.rpoplpush(queue, self.class.working_queue_name(queue))
         end

data/spec/base_reliable_fetch_spec.rb

@@ -39,14 +39,15 @@ describe Sidekiq::BaseReliableFetch do
     end
   end
 
-  describe '.bulk_requeue' do
+  describe '#bulk_requeue' do
+    let(:options) { { queues: %w[foo bar] } }
     let!(:queue1) { Sidekiq::Queue.new('foo') }
     let!(:queue2) { Sidekiq::Queue.new('bar') }
 
     it 'requeues the bulk' do
       uow = described_class::UnitOfWork
       jobs = [ uow.new('queue:foo', job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
-      described_class.bulk_requeue(jobs, queues: [])
+      described_class.new(options).bulk_requeue(jobs, nil)
 
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
@@ -56,7 +57,7 @@ describe Sidekiq::BaseReliableFetch do
       uow = described_class::UnitOfWork
       interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
       jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
-      described_class.bulk_requeue(jobs, queues: [])
+      described_class.new(options).bulk_requeue(jobs, nil)
 
       expect(queue1.size).to eq 1
       expect(queue2.size).to eq 1
@@ -69,7 +70,7 @@ describe Sidekiq::BaseReliableFetch do
       uow = described_class::UnitOfWork
       interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
       jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
-      described_class.bulk_requeue(jobs, queues: [])
+      described_class.new(options).bulk_requeue(jobs, nil)
 
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
@@ -80,7 +81,7 @@ describe Sidekiq::BaseReliableFetch do
   end
 
   it 'sets heartbeat' do
-    config = double(:sidekiq_config, options: {})
+    config = double(:sidekiq_config, options: { queues: %w[foo bar] })
 
     heartbeat_thread = described_class.setup_reliable_fetch!(config)
 

data/spec/fetch_shared_examples.rb

@@ -5,102 +5,16 @@ shared_examples 'a Sidekiq fetcher' do
 
   describe '#retrieve_work' do
     let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
-    let(:fetcher) { described_class.new(queues: ['assigned']) }
-
-    it 'retrieves the job and puts it to working queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
-
-      uow = fetcher.retrieve_work
-
-      expect(working_queue_size('assigned')).to eq 1
-      expect(uow.queue_name).to eq 'assigned'
-      expect(uow.job).to eq job
-      expect(Sidekiq::Queue.new('assigned').size).to eq 0
-    end
-
-    it 'does not retrieve a job from foreign queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', job) }
-
-      expect(fetcher.retrieve_work).to be_nil
-    end
-
-    it 'requeues jobs from dead working queue with incremented interrupted_count' do
-      Sidekiq.redis do |conn|
-        conn.rpush(other_process_working_queue_name('assigned'), job)
-      end
-
-      expected_job = Sidekiq.load_json(job)
-      expected_job['interrupted_count'] = 1
-      expected_job = Sidekiq.dump_json(expected_job)
-
-      uow = fetcher.retrieve_work
-
-      expect(uow).to_not be_nil
-      expect(uow.job).to eq expected_job
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
-      end
-    end
-
-    it 'ignores working queue keys in unknown formats' do
-      # Add a spurious non-numeric char segment at the end; this simulates any other
-      # incorrect form in general
-      malformed_key = "#{other_process_working_queue_name('assigned')}:X"
-      Sidekiq.redis do |conn|
-        conn.rpush(malformed_key, job)
-      end
-
-      uow = fetcher.retrieve_work
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(malformed_key)).to eq 1
-      end
-    end
-
-    it 'requeues jobs from legacy dead working queue with incremented interrupted_count' do
-      Sidekiq.redis do |conn|
-        conn.rpush(legacy_other_process_working_queue_name('assigned'), job)
-      end
-
-      expected_job = Sidekiq.load_json(job)
-      expected_job['interrupted_count'] = 1
-      expected_job = Sidekiq.dump_json(expected_job)
-
-      uow = fetcher.retrieve_work
-
-      expect(uow).to_not be_nil
-      expect(uow.job).to eq expected_job
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(legacy_other_process_working_queue_name('assigned'))).to eq 0
-      end
-    end
-
-    it 'does not requeue jobs from live working queue' do
-      working_queue = live_other_process_working_queue_name('assigned')
-
-      Sidekiq.redis do |conn|
-        conn.rpush(working_queue, job)
-      end
-
-      uow = fetcher.retrieve_work
-
-      expect(uow).to be_nil
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(working_queue)).to eq 1
-      end
-    end
+    let(:fetcher) { described_class.new(queues: queues) }
 
     it 'does not clean up orphaned jobs more than once per cleanup interval' do
       Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
 
-      expect(described_class).to receive(:clean_working_queues!).once
+      expect(fetcher).to receive(:clean_working_queues!).once
 
       threads = 10.times.map do
         Thread.new do
-          described_class.new(queues: ['assigned']).retrieve_work
+          fetcher.retrieve_work
         end
       end
 
@@ -133,13 +47,34 @@ shared_examples 'a Sidekiq fetcher' do
       expect(jobs).to include 'this_job_should_not_stuck'
     end
 
-    context 'with namespaced queues' do
-      let (:queue) { 'namespace:assigned' }
+    shared_examples "basic queue handling" do |queue|
       let (:fetcher) { described_class.new(queues: [queue]) }
 
-      it 'requeues jobs from dead namespaced working queue with incremented interrupted_count' do
+      it 'retrieves the job and puts it to working queue' do
+        Sidekiq.redis { |conn| conn.rpush("queue:#{queue}", job) }
+
+        uow = fetcher.retrieve_work
+
+        expect(working_queue_size(queue)).to eq 1
+        expect(uow.queue_name).to eq queue
+        expect(uow.job).to eq job
+        expect(Sidekiq::Queue.new(queue).size).to eq 0
+      end
+
+      it 'does not retrieve a job from foreign queue' do
+        Sidekiq.redis { |conn| conn.rpush("'queue:#{queue}:not", job) }
+        expect(fetcher.retrieve_work).to be_nil
+
+        Sidekiq.redis { |conn| conn.rpush("'queue:not_#{queue}", job) }
+        expect(fetcher.retrieve_work).to be_nil
+
+        Sidekiq.redis { |conn| conn.rpush("'queue:random_name", job) }
+        expect(fetcher.retrieve_work).to be_nil
+      end
+
+      it 'requeues jobs from legacy dead working queue with incremented interrupted_count' do
         Sidekiq.redis do |conn|
-          conn.rpush(other_process_working_queue_name(queue), job)
+          conn.rpush(legacy_other_process_working_queue_name(queue), job)
         end
 
         expected_job = Sidekiq.load_json(job)
@@ -152,32 +87,26 @@ shared_examples 'a Sidekiq fetcher' do
         expect(uow.job).to eq expected_job
 
         Sidekiq.redis do |conn|
-          expect(conn.llen(other_process_working_queue_name(queue))).to eq 0
+          expect(conn.llen(legacy_other_process_working_queue_name(queue))).to eq 0
         end
       end
 
-      it 'does not requeue jobs in a namespaced queue from live working queue' do
-        working_queue = live_other_process_working_queue_name(queue)
-
+      it 'ignores working queue keys in unknown formats' do
+        # Add a spurious non-numeric char segment at the end; this simulates any other
+        # incorrect form in general
+        malformed_key = "#{other_process_working_queue_name(queue)}:X"
         Sidekiq.redis do |conn|
-          conn.rpush(working_queue, job)
+          conn.rpush(malformed_key, job)
         end
 
         uow = fetcher.retrieve_work
 
-        expect(uow).to be_nil
-
         Sidekiq.redis do |conn|
-          expect(conn.llen(working_queue)).to eq 1
+          expect(conn.llen(malformed_key)).to eq 1
         end
       end
-    end
-
-    context 'with deeper namespaced queues' do
-      let (:queue) { 'deep:namespace:assigned' }
-      let (:fetcher) { described_class.new(queues: [queue]) }
 
-      it 'requeues jobs from dead namespaced working queue with incremented interrupted_count' do
+      it 'requeues jobs from dead working queue with incremented interrupted_count' do
         Sidekiq.redis do |conn|
           conn.rpush(other_process_working_queue_name(queue), job)
         end
@@ -196,7 +125,7 @@ shared_examples 'a Sidekiq fetcher' do
         end
       end
 
-      it 'does not requeue jobs in a deeper namespaced queue from live working queue' do
+      it 'does not requeue jobs from live working queue' do
         working_queue = live_other_process_working_queue_name(queue)
 
         Sidekiq.redis do |conn|
@@ -213,6 +142,12 @@ shared_examples 'a Sidekiq fetcher' do
       end
     end
 
+    context 'with various queues' do
+      %w[assigned namespace:assigned namespace:deeper:assigned].each do |queue|
+        it_behaves_like "basic queue handling", queue
+      end
+    end
+
     context 'with short cleanup interval' do
       let(:short_interval) { 1 }
       let(:fetcher) { described_class.new(queues: queues, lease_interval: short_interval, cleanup_interval: short_interval) }
@@ -243,7 +178,6 @@ def legacy_other_process_working_queue_name(queue)
   "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
 end
 
-
 def other_process_working_queue_name(queue)
   "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}:#{::SecureRandom.hex(6)}"
 end

data/tests/reliability/reliability_test.rb

@@ -57,7 +57,7 @@ end
 def spawn_workers
   pids = []
   NUMBER_OF_WORKERS.times do
-    pids << spawn('sidekiq -r ./config.rb')
+    pids << spawn('sidekiq -q default -q low -q high -r ./config.rb')
   end
 
   pids

data/tests/reliability/worker.rb

@@ -8,19 +8,7 @@ class ReliabilityTestWorker
     sleep 1
 
     Sidekiq.redis do |redis|
-      redis.lpush(REDIS_FINISHED_LIST, get_sidekiq_job_id)
+      redis.lpush(REDIS_FINISHED_LIST, jid)
     end
   end
-
-  def get_sidekiq_job_id
-    context_data = Thread.current[:sidekiq_context]&.first
-
-    return unless context_data
-
-    index = context_data.index('JID-')
-
-    return unless index
-
-    context_data[index + 4..-1]
-  end
 end

data/tests/support/utils.rb

@@ -11,7 +11,7 @@ def spawn_workers(number)
   pids = []
 
   number.times do
-    pids << spawn('sidekiq -r ./config.rb')
+    pids << spawn('sidekiq -q default -q high -q low -r ./config.rb')
   end
 
   pids

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.8.0
 platform: ruby
 authors:
 - TEA
 - GitLab
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-24 00:00:00.000000000 Z
+date: 2021-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6.1'
 description: Redis reliable queue pattern implemented in Sidekiq
 email: valery@gitlab.com
 executables: []
@@ -63,7 +63,7 @@ homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -79,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq
 test_files: []