gitlab-sidekiq-fetcher 0.7.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efdc9461358b538f2b0692cc179639b802a6f0bf4959b73e1d0821a4c40f0713
-  data.tar.gz: ccbe252394f22e6fafb198ddba9481e23776e7cfa2848807e1e5c34a5269c76d
+  metadata.gz: 7f264e7d628cba58e996b5e2428e81a6e2d6a4921738bd5ade6fd9c3637fa8ba
+  data.tar.gz: 588f7a6e9a5d24d229bc6dbc7d0a6ca4cb9ec96a517bd9015659ed5b3c67eb8e
 SHA512:
-  metadata.gz: 010da1750797c367b01cd8e1d8df281fa70759e9035f0ebaea71f53349f3ebbe54bb04dea74694d50bb595938b049733119ffd5ee4c85d5fd42d719e71a07809
-  data.tar.gz: d9f89a5dc1e6d6117df1f3704159ea79160a038d8a7a47939d1c772f07c4f3be2426389cd63d14a0ecf469035cc583ba3ea943cf5a14c2232ee3cb44bf09b21c
+  metadata.gz: 955716fcccc6be01e941e1e366268c78d8179577d007e324bb68a56c7741ee6045e3fc7e66dab1b58d794f5c22ffc2c2af400c95dbe61ad697e8e5685a753eae
+  data.tar.gz: df0fd2c4c72a8401e2f0ac7f8eb3e7f5f47d1fa75da4904e0a0b67c4d80a57a47334e4bdf72728e830f756bbbb6b004412067ad36e8a66a9605471ddc010898f

data/.gitlab-ci.yml

@@ -1,4 +1,8 @@
-image: "ruby:2.5"
+variables:
+  RUBY_VERSION: "2.7"
+
+default:
+  image: ruby:${RUBY_VERSION}
 
 before_script:
   - ruby -v
@@ -21,6 +25,10 @@ rspec:
     when: always
     paths:
       - coverage/
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 .integration:
   stage: test
@@ -29,6 +37,10 @@ rspec:
     - bundle exec ruby reliability_test.rb
   services:
     - redis:alpine
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 integration_semi:
   extends: .integration
@@ -53,6 +65,10 @@ kill_interruption:
     - bundle exec ruby test_kill_signal.rb
   services:
     - redis:alpine
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 term_interruption:
   stage: test
@@ -61,6 +77,10 @@ term_interruption:
     - bundle exec ruby test_term_signal.rb
   services:
     - redis:alpine
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 # rubocop:
 #   script:

data/CONTRIBUTING.md

@@ -0,0 +1,41 @@
+## Developer Certificate of Origin and License
+
+By contributing to GitLab B.V., you accept and agree to the following terms and
+conditions for your present and future contributions submitted to GitLab B.V.
+Except for the license granted herein to GitLab B.V. and recipients of software
+distributed by GitLab B.V., you reserve all right, title, and interest in and to
+your Contributions.
+
+All contributions are subject to the Developer Certificate of Origin and license set out at [docs.gitlab.com/ce/legal/developer_certificate_of_origin](https://docs.gitlab.com/ce/legal/developer_certificate_of_origin).
+
+_This notice should stay as the first item in the CONTRIBUTING.md file._
+
+## Code of conduct
+
+As contributors and maintainers of this project, we pledge to respect all people
+who contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual
+language or imagery, derogatory comments or personal attacks, trolling, public
+or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct. Project maintainers who do not follow the
+Code of Conduct may be removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior can be
+reported by emailing contact@gitlab.com.
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.1.0,
+available at [https://contributor-covenant.org/version/1/1/0/](https://contributor-covenant.org/version/1/1/0/).
+

data/Gemfile

@@ -4,9 +4,11 @@ source "https://rubygems.org"
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
+gemspec
+
 group :test do
   gem "rspec", '~> 3'
   gem "pry"
-  gem "sidekiq", '~> 6.1'
   gem 'simplecov', require: false
+  gem 'stub_env', '~> 1.0'
 end

data/Gemfile.lock

@@ -1,17 +1,24 @@
+PATH
+  remote: .
+  specs:
+    gitlab-sidekiq-fetcher (0.8.0)
+      json (>= 2.5)
+      sidekiq (~> 6.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    connection_pool (2.2.3)
+    connection_pool (2.3.0)
     diff-lcs (1.3)
     docile (1.3.1)
-    json (2.1.0)
+    json (2.5.1)
     method_source (0.9.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.2.3)
-    redis (4.2.1)
+    rack (2.2.4)
+    redis (4.8.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -25,24 +32,27 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    sidekiq (6.1.0)
-      connection_pool (>= 2.2.2)
+    sidekiq (6.5.7)
+      connection_pool (>= 2.2.5)
       rack (~> 2.0)
-      redis (>= 4.2.0)
+      redis (>= 4.5.0, < 5)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    stub_env (1.0.4)
+      rspec (>= 2.0, < 4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  gitlab-sidekiq-fetcher!
   pry
   rspec (~> 3)
-  sidekiq (~> 6.1)
   simplecov
+  stub_env (~> 1.0)
 
 BUNDLED WITH
-   1.17.2
+   2.3.24

data/README.md

@@ -8,6 +8,8 @@ It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
 
 **IMPORTANT NOTE:** Since version `0.7.0` this gem works only with `sidekiq >= 6.1` (which introduced Fetch API breaking changes). Please use version `~> 0.5` if you use older version of the `sidekiq` .
 
+**UPGRADE NOTE:** If upgrading from 0.7.0, strongly consider a full deployed step on 0.7.1 before 0.8.0; that fixes a bug in the queue name validation that will hit if sidekiq nodes running 0.7.0 see working queues named by 0.8.0.  See https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/-/merge_requests/22
+
 There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
 semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
 However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.
@@ -44,11 +46,11 @@ Sidekiq.configure_server do |config|
 end
 ```
 
-There is an additional parameter `config.options[:semi_reliable_fetch]` you can use to switch between two strategies:
+There is an additional parameter `config[:semi_reliable_fetch]` you can use to switch between two strategies:
 
 ```ruby
 Sidekiq.configure_server do |config|
-  config.options[:semi_reliable_fetch] = true # Default value is false
+  config[:semi_reliable_fetch] = true # Default value is false
 
   Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
 end

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.7.1'
+  s.version       = '0.9.0'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'
@@ -11,4 +11,5 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files`.split($\)
   s.test_files    = []
   s.add_dependency 'sidekiq', '~> 6.1'
+  s.add_runtime_dependency 'json', '>= 2.5'
 end

data/lib/sidekiq/base_reliable_fetch.rb

@@ -21,6 +21,10 @@ module Sidekiq
     # How much time a job can be interrupted
     DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION = 3
 
+    # Regexes for matching working queue keys
+    WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*:[0-9a-f]*)\z/.freeze
+    LEGACY_WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*)\z/.freeze
+
     UnitOfWork = Struct.new(:queue, :job) do
       def acknowledge
         Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
@@ -41,13 +45,15 @@ module Sidekiq
     end
 
     def self.setup_reliable_fetch!(config)
-      fetch_strategy = if config.options[:semi_reliable_fetch]
+      config = config.options unless config.respond_to?(:[])
+
+      fetch_strategy = if config[:semi_reliable_fetch]
                          Sidekiq::SemiReliableFetch
                        else
                          Sidekiq::ReliableFetch
                        end
 
-      config.options[:fetch] = fetch_strategy.new(config.options)
+      config[:fetch] = fetch_strategy.new(config)
 
       Sidekiq.logger.info('GitLab reliable fetch activated!')
 
@@ -70,32 +76,36 @@ module Sidekiq
       end
     end
 
-    def self.pid
-      @pid ||= ::Process.pid
+    def self.hostname
+      Socket.gethostname
+    end
+
+    def self.process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
     end
 
-    def self.hostname
-      @hostname ||= Socket.gethostname
+    def self.identity
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
     def self.heartbeat
       Sidekiq.redis do |conn|
-        conn.set(heartbeat_key(hostname, pid), 1, ex: HEARTBEAT_LIFESPAN)
+        conn.set(heartbeat_key(identity), 1, ex: HEARTBEAT_LIFESPAN)
       end
 
-      Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
+      Sidekiq.logger.debug("Heartbeat for #{identity}")
     end
 
-    def self.worker_dead?(hostname, pid, conn)
-      !conn.get(heartbeat_key(hostname, pid))
+    def self.worker_dead?(identity, conn)
+      !conn.get(heartbeat_key(identity))
     end
 
-    def self.heartbeat_key(hostname, pid)
-      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
+    def self.heartbeat_key(identity)
+      "reliable-fetcher-heartbeat-#{identity.gsub(':', '-')}"
     end
 
     def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{identity}"
     end
 
     attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
@@ -105,6 +115,8 @@ module Sidekiq
     def initialize(options)
       raise ArgumentError, 'missing queue list' unless options[:queues]
 
+      @config = options
+      @interrupted_set = Sidekiq::InterruptedSet.new
       @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
       @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
       @last_try_to_take_lease_at = 0
@@ -166,6 +178,19 @@ module Sidekiq
       )
     end
 
+    def extract_queue_and_identity(key)
+      # New identity format is "{hostname}:{pid}:{randomhex}
+      # Old identity format is "{hostname}:{pid}"
+      # Queue names may also have colons (namespaced).
+      # Expressing this in a single regex is unreadable
+
+      # Test the newer expected format first, only checking the older if necessary
+      original_queue, identity = key.scan(WORKING_QUEUE_REGEX).flatten
+      return original_queue, identity unless original_queue.nil? || identity.nil?
+
+      key.scan(LEGACY_WORKING_QUEUE_REGEX).flatten
+    end
+
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
     def clean_working_queues!
@@ -173,19 +198,16 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
-          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
+          original_queue, identity = extract_queue_and_identity(key)
 
-          next if hostname.nil? || pid.nil?
+          next if original_queue.nil? || identity.nil?
 
-          clean_working_queue!(key) if self.class.worker_dead?(hostname, pid, conn)
+          clean_working_queue!(original_queue, key) if self.class.worker_dead?(identity, conn)
         end
       end
     end
 
-    def clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
-
+    def clean_working_queue!(original_queue, working_queue)
       Sidekiq.redis do |conn|
         while job = conn.rpop(working_queue)
           preprocess_interrupted_job(job, original_queue)
@@ -207,7 +229,7 @@ module Sidekiq
       rescue NameError
       end
 
-      max_retries_after_interruption ||= Sidekiq.options[:max_retries_after_interruption]
+      max_retries_after_interruption ||= @config[:max_retries_after_interruption]
       max_retries_after_interruption ||= DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION
       max_retries_after_interruption
     end
@@ -220,7 +242,7 @@ module Sidekiq
       )
 
       job = Sidekiq.dump_json(msg)
-      Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
+      @interrupted_set.put(job, connection: multi_connection)
     end
 
     # Yield block with an existing connection or creates another one

data/lib/sidekiq/interrupted_set.rb

@@ -37,11 +37,15 @@ module Sidekiq
     end
 
     def self.max_jobs
-      Sidekiq.options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
+      options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
     end
 
     def self.timeout
-      Sidekiq.options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
+      options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
+    end
+
+    def self.options
+      Sidekiq.respond_to?(:[]) ? Sidekiq : Sidekiq.options
     end
   end
 end

data/lib/sidekiq/semi_reliable_fetch.rb

@@ -5,14 +5,14 @@ module Sidekiq
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down. This constant is only used
     # for semi-reliable fetch.
-    SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
+    DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
 
     def initialize(options)
       super
 
       if strictly_ordered_queues
         @queues = @queues.uniq
-        @queues << SEMI_RELIABLE_FETCH_TIMEOUT
+        @queues << { timeout: semi_reliable_fetch_timeout }
       end
     end
 
@@ -36,9 +36,13 @@ module Sidekiq
         @queues
       else
         queues = @queues.shuffle.uniq
-        queues << SEMI_RELIABLE_FETCH_TIMEOUT
+        queues << { timeout: semi_reliable_fetch_timeout }
         queues
       end
     end
+
+    def semi_reliable_fetch_timeout
+      @semi_reliable_fetch_timeout ||= ENV['SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT']&.to_i || DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT
+    end
   end
 end

data/spec/base_reliable_fetch_spec.rb

@@ -65,7 +65,7 @@ describe Sidekiq::BaseReliableFetch do
     end
 
     it 'does not put jobs into interrupted queue if it is disabled' do
-      Sidekiq.options[:max_retries_after_interruption] = -1
+      options[:max_retries_after_interruption] = -1
 
       uow = described_class::UnitOfWork
       interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
@@ -75,8 +75,6 @@ describe Sidekiq::BaseReliableFetch do
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
       expect(Sidekiq::InterruptedSet.new.size).to eq 0
-
-      Sidekiq.options[:max_retries_after_interruption] = 3
     end
   end
 
@@ -88,7 +86,7 @@ describe Sidekiq::BaseReliableFetch do
     Sidekiq.redis do |conn|
       sleep 0.2 # Give the time to heartbeat thread to make a loop
 
-      heartbeat_key = described_class.heartbeat_key(Socket.gethostname, ::Process.pid)
+      heartbeat_key = described_class.heartbeat_key(described_class.identity)
       heartbeat = conn.get(heartbeat_key)
 
       expect(heartbeat).not_to be_nil

data/spec/fetch_shared_examples.rb

@@ -7,111 +7,163 @@ shared_examples 'a Sidekiq fetcher' do
     let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
     let(:fetcher) { described_class.new(queues: queues) }
 
-    it 'retrieves the job and puts it to working queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
-
-      uow = fetcher.retrieve_work
+    it 'does not clean up orphaned jobs more than once per cleanup interval' do
+      Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
 
-      expect(working_queue_size('assigned')).to eq 1
-      expect(uow.queue_name).to eq 'assigned'
-      expect(uow.job).to eq job
-      expect(Sidekiq::Queue.new('assigned').size).to eq 0
-    end
+      expect(fetcher).to receive(:clean_working_queues!).once
 
-    it 'does not retrieve a job from foreign queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', job) }
+      threads = 10.times.map do
+        Thread.new do
+          fetcher.retrieve_work
+        end
+      end
 
-      expect(fetcher.retrieve_work).to be_nil
+      threads.map(&:join)
     end
 
-    it 'requeues jobs from dead working queue with incremented interrupted_count' do
+    it 'retrieves by order when strictly order is enabled' do
+      fetcher = described_class.new(strict: true, queues: ['first', 'second'])
+
       Sidekiq.redis do |conn|
-        conn.rpush(other_process_working_queue_name('assigned'), job)
+        conn.rpush('queue:first', ['msg3', 'msg2', 'msg1'])
+        conn.rpush('queue:second', 'msg4')
       end
 
-      expected_job = Sidekiq.load_json(job)
-      expected_job['interrupted_count'] = 1
-      expected_job = Sidekiq.dump_json(expected_job)
+      jobs = (1..4).map { fetcher.retrieve_work.job }
 
-      uow = fetcher.retrieve_work
+      expect(jobs).to eq ['msg1', 'msg2', 'msg3', 'msg4']
+    end
 
-      expect(uow.job).to eq expected_job
+    it 'does not starve any queue when queues are not strictly ordered' do
+      fetcher = described_class.new(queues: ['first', 'second'])
 
       Sidekiq.redis do |conn|
-        expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
+        conn.rpush('queue:first', (1..200).map { |i| "msg#{i}" })
+        conn.rpush('queue:second', 'this_job_should_not_stuck')
       end
+
+      jobs = (1..100).map { fetcher.retrieve_work.job }
+
+      expect(jobs).to include 'this_job_should_not_stuck'
     end
 
-    it 'ignores working queue keys in unknown formats' do
-      # Add a spurious non-numeric char segment at the end; this simulates any other
-      # incorrect form in general
-      malformed_key = "#{other_process_working_queue_name('assigned')}:X"
-      Sidekiq.redis do |conn|
-        conn.rpush(malformed_key, job)
-      end
+    shared_examples "basic queue handling" do |queue|
+      let (:fetcher) { described_class.new(queues: [queue]) }
 
-      uow = fetcher.retrieve_work
+      it 'retrieves the job and puts it to working queue' do
+        Sidekiq.redis { |conn| conn.rpush("queue:#{queue}", job) }
 
-      Sidekiq.redis do |conn|
-        expect(conn.llen(malformed_key)).to eq 1
+        uow = fetcher.retrieve_work
+
+        expect(working_queue_size(queue)).to eq 1
+        expect(uow.queue_name).to eq queue
+        expect(uow.job).to eq job
+        expect(Sidekiq::Queue.new(queue).size).to eq 0
       end
-    end
 
+      it 'does not retrieve a job from foreign queue' do
+        Sidekiq.redis { |conn| conn.rpush("'queue:#{queue}:not", job) }
+        expect(fetcher.retrieve_work).to be_nil
 
-    it 'does not requeue jobs from live working queue' do
-      working_queue = live_other_process_working_queue_name('assigned')
+        Sidekiq.redis { |conn| conn.rpush("'queue:not_#{queue}", job) }
+        expect(fetcher.retrieve_work).to be_nil
 
-      Sidekiq.redis do |conn|
-        conn.rpush(working_queue, job)
+        Sidekiq.redis { |conn| conn.rpush("'queue:random_name", job) }
+        expect(fetcher.retrieve_work).to be_nil
       end
 
-      uow = fetcher.retrieve_work
+      it 'requeues jobs from legacy dead working queue with incremented interrupted_count' do
+        Sidekiq.redis do |conn|
+          conn.rpush(legacy_other_process_working_queue_name(queue), job)
+        end
 
-      expect(uow).to be_nil
+        expected_job = Sidekiq.load_json(job)
+        expected_job['interrupted_count'] = 1
+        expected_job = Sidekiq.dump_json(expected_job)
 
-      Sidekiq.redis do |conn|
-        expect(conn.llen(working_queue)).to eq 1
+        uow = fetcher.retrieve_work
+
+        expect(uow).to_not be_nil
+        expect(uow.job).to eq expected_job
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(legacy_other_process_working_queue_name(queue))).to eq 0
+        end
       end
-    end
 
-    it 'does not clean up orphaned jobs more than once per cleanup interval' do
-      Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
+      it 'ignores working queue keys in unknown formats' do
+        # Add a spurious non-numeric char segment at the end; this simulates any other
+        # incorrect form in general
+        malformed_key = "#{other_process_working_queue_name(queue)}:X"
+        Sidekiq.redis do |conn|
+          conn.rpush(malformed_key, job)
+        end
 
-      expect(fetcher).to receive(:clean_working_queues!).once
+        uow = fetcher.retrieve_work
 
-      threads = 10.times.map do
-        Thread.new do
-          fetcher.retrieve_work
+        Sidekiq.redis do |conn|
+          expect(conn.llen(malformed_key)).to eq 1
         end
       end
 
-      threads.map(&:join)
-    end
+      it 'requeues jobs from dead working queue with incremented interrupted_count' do
+        Sidekiq.redis do |conn|
+          conn.rpush(other_process_working_queue_name(queue), job)
+        end
 
-    it 'retrieves by order when strictly order is enabled' do
-      fetcher = described_class.new(strict: true, queues: ['first', 'second'])
+        expected_job = Sidekiq.load_json(job)
+        expected_job['interrupted_count'] = 1
+        expected_job = Sidekiq.dump_json(expected_job)
 
-      Sidekiq.redis do |conn|
-        conn.rpush('queue:first', ['msg3', 'msg2', 'msg1'])
-        conn.rpush('queue:second', 'msg4')
+        uow = fetcher.retrieve_work
+
+        expect(uow).to_not be_nil
+        expect(uow.job).to eq expected_job
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(other_process_working_queue_name(queue))).to eq 0
+        end
       end
 
-      jobs = (1..4).map { fetcher.retrieve_work.job }
+      it 'does not requeue jobs from live working queue' do
+        working_queue = live_other_process_working_queue_name(queue)
 
-      expect(jobs).to eq ['msg1', 'msg2', 'msg3', 'msg4']
-    end
+        Sidekiq.redis do |conn|
+          conn.rpush(working_queue, job)
+        end
 
-    it 'does not starve any queue when queues are not strictly ordered' do
-      fetcher = described_class.new(queues: ['first', 'second'])
+        uow = fetcher.retrieve_work
 
-      Sidekiq.redis do |conn|
-        conn.rpush('queue:first', (1..200).map { |i| "msg#{i}" })
-        conn.rpush('queue:second', 'this_job_should_not_stuck')
+        expect(uow).to be_nil
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(working_queue)).to eq 1
+        end
       end
+    end
 
-      jobs = (1..100).map { fetcher.retrieve_work.job }
+    context 'with various queues' do
+      %w[assigned namespace:assigned namespace:deeper:assigned].each do |queue|
+        it_behaves_like "basic queue handling", queue
+      end
+    end
 
-      expect(jobs).to include 'this_job_should_not_stuck'
+    context 'with short cleanup interval' do
+      let(:short_interval) { 1 }
+      let(:fetcher) { described_class.new(queues: queues, lease_interval: short_interval, cleanup_interval: short_interval) }
+
+      it 'requeues when there is no heartbeat' do
+        Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
+        # Use of retrieve_work twice with a sleep ensures we have exercised the
+        # `identity` method to create the working queue key name and that it
+        # matches the patterns used in the cleanup
+        uow = fetcher.retrieve_work
+        sleep(short_interval + 1)
+        uow = fetcher.retrieve_work
+
+        # Will only receive a UnitOfWork if the job was detected as failed and requeued
+        expect(uow).to_not be_nil
+      end
     end
   end
 end
@@ -122,17 +174,22 @@ def working_queue_size(queue_name)
   end
 end
 
-def other_process_working_queue_name(queue)
+def legacy_other_process_working_queue_name(queue)
   "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
 end
 
+def other_process_working_queue_name(queue)
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}:#{::SecureRandom.hex(6)}"
+end
+
 def live_other_process_working_queue_name(queue)
   pid = ::Process.pid + 1
   hostname = Socket.gethostname
+  nonce = SecureRandom.hex(6)
 
   Sidekiq.redis do |conn|
-    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key(hostname, pid), 1)
+    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key("#{hostname}-#{pid}-#{nonce}"), 1)
   end
 
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}"
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}:#{nonce}"
 end

data/spec/semi_reliable_fetch_spec.rb

@@ -5,4 +5,39 @@ require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::SemiReliableFetch do
   include_examples 'a Sidekiq fetcher'
+
+  describe '#retrieve_work' do
+    context 'timeout config' do
+      let(:queues) { ['stuff_to_do'] }
+      let(:fetcher) { described_class.new(queues: queues) }
+
+      before do
+        stub_env('SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT', timeout)
+      end
+
+      context 'when the timeout is not configured' do
+        let(:timeout) { nil }
+
+        it 'brpops with the default timeout timeout' do
+          Sidekiq.redis do |connection|
+            expect(connection).to receive(:brpop).with("queue:stuff_to_do", { timeout: 2 }).once.and_call_original
+
+            fetcher.retrieve_work
+          end
+        end
+      end
+
+      context 'when the timeout is set in the env' do
+        let(:timeout) { '5' }
+
+        it 'brpops with the default timeout timeout' do
+          Sidekiq.redis do |connection|
+            expect(connection).to receive(:brpop).with("queue:stuff_to_do", { timeout: 5 }).once.and_call_original
+
+            fetcher.retrieve_work
+          end
+        end
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,8 +1,8 @@
 require 'sidekiq'
-require 'sidekiq/util'
 require 'sidekiq/api'
 require 'pry'
 require 'simplecov'
+require 'stub_env'
 
 SimpleCov.start
 
@@ -29,6 +29,7 @@ Sidekiq.logger.level = Logger::ERROR
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|
+  config.include StubEnv::Helpers
   # rspec-expectations config goes here. You can use an alternate
   # assertion/expectation library such as wrong or the stdlib/minitest
   # assertions if you prefer.

data/tests/README.md

@@ -1,7 +1,7 @@
 # How to run reliability tests
 
 ```
-cd reliability_test
+cd tests/reliability
 bundle exec ruby reliability_test.rb
 ```
 

data/tests/interruption/config.rb

@@ -7,13 +7,13 @@ TEST_CLEANUP_INTERVAL = 20
 TEST_LEASE_INTERVAL = 5
 
 Sidekiq.configure_server do |config|
-  config.options[:semi_reliable_fetch] = true
+  config[:semi_reliable_fetch] = true
 
   # We need to override these parameters to not wait too long
   # The default values are good for production use only
   # These will be ignored for :basic
-  config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
-  config.options[:lease_interval] = TEST_LEASE_INTERVAL
+  config[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+  config[:lease_interval] = TEST_LEASE_INTERVAL
 
   Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
 end

data/tests/reliability/config.rb

@@ -16,13 +16,13 @@ WAIT_CLEANUP = TEST_CLEANUP_INTERVAL +
 
 Sidekiq.configure_server do |config|
   if %i[semi reliable].include?(JOB_FETCHER)
-    config.options[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
+    config[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
 
     # We need to override these parameters to not wait too long
     # The default values are good for production use only
     # These will be ignored for :basic
-    config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
-    config.options[:lease_interval] = TEST_LEASE_INTERVAL
+    config[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+    config[:lease_interval] = TEST_LEASE_INTERVAL
 
     Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
   end

data/tests/reliability/reliability_test.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
-require 'sidekiq/util'
 require 'sidekiq/cli'
 require_relative 'config'
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.9.0
 platform: ruby
 authors:
 - TEA
 - GitLab
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-18 00:00:00.000000000 Z
+date: 2022-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -25,6 +25,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.5'
 description: Redis reliable queue pattern implemented in Sidekiq
 email: valery@gitlab.com
 executables: []
@@ -34,6 +48,7 @@ files:
 - ".gitignore"
 - ".gitlab-ci.yml"
 - ".rspec"
+- CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -63,7 +78,7 @@ homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -78,8 +93,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq
 test_files: []