gitlab-sidekiq-fetcher 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 477650a08755f00beb453c4867a270fa8469a83a372b5b09ba1c030885899699
-  data.tar.gz: da3fcf3f0b67dd3f71c73ea80d09666e82fa9360cdf5cbf6c8b8e5668b85bfb5
+  metadata.gz: eb384451139d1638854cd94a673b12e0f7fd5a42c663535da0145fd153ba4c51
+  data.tar.gz: 43851081b6406fe8a876824ef034c548665382707db0aaf41c84c186a98419b0
 SHA512:
-  metadata.gz: b4d03a71a6c00e2fa55affd4af5633c2c1ef58093c8bd82b95d4a1603523d940e35b8db4643becf20236224ef065fc879b70585933569efa893e0e05481bffb9
-  data.tar.gz: 4fac10a26e3507c70927ad8689c2d8903d452cc9ed25a6522dc5c2c7e5e7fcb13d8412013eadb435a9a2feecab51130ad516553c4bbb49c72b2b36827f75c3cf
+  metadata.gz: 617032e226ffc898b1c411b0f7598169272bc8521156b9eadcb35b48a139cd854f1b12a2fd98e7fd174128727f0b8c60917780cd2b6dec82cb1370a6260b20ee
+  data.tar.gz: 5e56e3840a5f0a4f437ec298de7b68ef796dcfa7d31e69117398ceb22b4ed07fbb4eac4e711b344f5134fd748f9afa476d0d4ffc216cc02cd4eec5b9db90a9c2

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.5.4'
+  s.version       = '0.5.5'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -21,6 +21,10 @@ module Sidekiq
     # How much time a job can be interrupted
     DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION = 3
 
+    # Regexes for matching working queue keys
+    WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*:[0-9a-f]*)\z/.freeze
+    LEGACY_WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*)\z/.freeze
+
     UnitOfWork = Struct.new(:queue, :job) do
       def acknowledge
         Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
@@ -123,12 +127,17 @@ module Sidekiq
       end
     end
 
-    def self.valid_identity_format?(identity)
-      # New format is "{hostname}:{pid}:{randomhex}
-      # Old format is "{hostname}:{pid}"
+    def self.extract_queue_and_identity(key)
+      # New identity format is "{hostname}:{pid}:{randomhex}
+      # Old identity format is "{hostname}:{pid}"
+      # Queue names may also have colons (namespaced).
+      # Expressing this in a single regex is unreadable
+
+      # Test the newer expected format first, only checking the older if necessary
+      original_queue, identity = key.scan(WORKING_QUEUE_REGEX).flatten
+      return original_queue, identity unless original_queue.nil? || identity.nil?
 
-      # Test the newer format first, only checking the older if necessary
-      identity.match(/[^:]*:[0-9]*:[0-9a-f]*\z/) || identity.match(/([^:]*):([0-9]*)\z/)
+      key.scan(LEGACY_WORKING_QUEUE_REGEX).flatten
     end
 
     # Detect "old" jobs and requeue them because the worker they were assigned
@@ -138,9 +147,9 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          original_queue, identity = key.scan(/#{WORKING_QUEUE_PREFIX}:(queue:[^:]*):(.*)\z/).flatten
+          original_queue, identity = extract_queue_and_identity(key)
 
-          next unless valid_identity_format?(identity)
+          next if original_queue.nil? || identity.nil?
 
           clean_working_queue!(original_queue, key) if worker_dead?(identity, conn)
         end

data/spec/fetch_shared_examples.rb

@@ -133,6 +133,86 @@ shared_examples 'a Sidekiq fetcher' do
       expect(jobs).to include 'this_job_should_not_stuck'
     end
 
+    context 'with namespaced queues' do
+      let (:queue) { 'namespace:assigned' }
+      let (:fetcher) { described_class.new(queues: [queue]) }
+
+      it 'requeues jobs from dead namespaced working queue with incremented interrupted_count' do
+        Sidekiq.redis do |conn|
+          conn.rpush(other_process_working_queue_name(queue), job)
+        end
+
+        expected_job = Sidekiq.load_json(job)
+        expected_job['interrupted_count'] = 1
+        expected_job = Sidekiq.dump_json(expected_job)
+
+        uow = fetcher.retrieve_work
+
+        expect(uow).to_not be_nil
+        expect(uow.job).to eq expected_job
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(other_process_working_queue_name(queue))).to eq 0
+        end
+      end
+
+      it 'does not requeue jobs in a namespaced queue from live working queue' do
+        working_queue = live_other_process_working_queue_name(queue)
+
+        Sidekiq.redis do |conn|
+          conn.rpush(working_queue, job)
+        end
+
+        uow = fetcher.retrieve_work
+
+        expect(uow).to be_nil
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(working_queue)).to eq 1
+        end
+      end
+    end
+
+    context 'with deeper namespaced queues' do
+      let (:queue) { 'deep:namespace:assigned' }
+      let (:fetcher) { described_class.new(queues: [queue]) }
+
+      it 'requeues jobs from dead namespaced working queue with incremented interrupted_count' do
+        Sidekiq.redis do |conn|
+          conn.rpush(other_process_working_queue_name(queue), job)
+        end
+
+        expected_job = Sidekiq.load_json(job)
+        expected_job['interrupted_count'] = 1
+        expected_job = Sidekiq.dump_json(expected_job)
+
+        uow = fetcher.retrieve_work
+
+        expect(uow).to_not be_nil
+        expect(uow.job).to eq expected_job
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(other_process_working_queue_name(queue))).to eq 0
+        end
+      end
+
+      it 'does not requeue jobs in a deeper namespaced queue from live working queue' do
+        working_queue = live_other_process_working_queue_name(queue)
+
+        Sidekiq.redis do |conn|
+          conn.rpush(working_queue, job)
+        end
+
+        uow = fetcher.retrieve_work
+
+        expect(uow).to be_nil
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(working_queue)).to eq 1
+        end
+      end
+    end
+
     context 'with short cleanup interval' do
       let(:short_interval) { 1 }
       let(:fetcher) { described_class.new(queues: queues, lease_interval: short_interval, cleanup_interval: short_interval) }

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - TEA
 - GitLab
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-22 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -63,7 +63,7 @@ homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -79,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq
 test_files: []