gitlab-sidekiq-fetcher 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efdc9461358b538f2b0692cc179639b802a6f0bf4959b73e1d0821a4c40f0713
-  data.tar.gz: ccbe252394f22e6fafb198ddba9481e23776e7cfa2848807e1e5c34a5269c76d
+  metadata.gz: c7be23d59956ffa44288a1c870bcca66fd0119682f810325d71a3ebaa8b76e80
+  data.tar.gz: 013a7124f61044572ad93335e95c18357c60804dd89024d987485b2d87775787
 SHA512:
-  metadata.gz: 010da1750797c367b01cd8e1d8df281fa70759e9035f0ebaea71f53349f3ebbe54bb04dea74694d50bb595938b049733119ffd5ee4c85d5fd42d719e71a07809
-  data.tar.gz: d9f89a5dc1e6d6117df1f3704159ea79160a038d8a7a47939d1c772f07c4f3be2426389cd63d14a0ecf469035cc583ba3ea943cf5a14c2232ee3cb44bf09b21c
+  metadata.gz: 92653bc5f9b5729f4dd50a8243a869c20d9621a1a9d25c46d729e735895e0f2d4d940c5a766803f1a5fd908ab0d9340f27d255dbd99f31bab4923e2f539c1882
+  data.tar.gz: d763b8b0ee3c2522752130fac86b83e67e8513faf919dd361aad9896aac684809650959b1627dd38e0440343c063f95b807b67fc31a10217ce1f15c428759803

data/README.md

@@ -8,6 +8,8 @@ It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
 
 **IMPORTANT NOTE:** Since version `0.7.0` this gem works only with `sidekiq >= 6.1` (which introduced Fetch API breaking changes). Please use version `~> 0.5` if you use older version of the `sidekiq` .
 
+**UPGRADE NOTE:** If upgrading from 0.7.0, strongly consider a full deployed step on 0.7.1 before 0.8.0; that fixes a bug in the queue name validation that will hit if sidekiq nodes running 0.7.0 see working queues named by 0.8.0.  See https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/-/merge_requests/22
+
 There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
 semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
 However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.7.1'
+  s.version       = '0.8.0'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -21,6 +21,10 @@ module Sidekiq
     # How much time a job can be interrupted
     DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION = 3
 
+    # Regexes for matching working queue keys
+    WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*:[0-9a-f]*)\z/.freeze
+    LEGACY_WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*)\z/.freeze
+
     UnitOfWork = Struct.new(:queue, :job) do
       def acknowledge
         Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
@@ -70,32 +74,36 @@ module Sidekiq
       end
     end
 
-    def self.pid
-      @pid ||= ::Process.pid
+    def self.hostname
+      Socket.gethostname
     end
 
-    def self.hostname
-      @hostname ||= Socket.gethostname
+    def self.process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def self.identity
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
     def self.heartbeat
       Sidekiq.redis do |conn|
-        conn.set(heartbeat_key(hostname, pid), 1, ex: HEARTBEAT_LIFESPAN)
+        conn.set(heartbeat_key(identity), 1, ex: HEARTBEAT_LIFESPAN)
       end
 
-      Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
+      Sidekiq.logger.debug("Heartbeat for #{identity}")
     end
 
-    def self.worker_dead?(hostname, pid, conn)
-      !conn.get(heartbeat_key(hostname, pid))
+    def self.worker_dead?(identity, conn)
+      !conn.get(heartbeat_key(identity))
     end
 
-    def self.heartbeat_key(hostname, pid)
-      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
+    def self.heartbeat_key(identity)
+      "reliable-fetcher-heartbeat-#{identity.gsub(':', '-')}"
     end
 
     def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{identity}"
     end
 
     attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
@@ -166,6 +174,19 @@ module Sidekiq
       )
     end
 
+    def extract_queue_and_identity(key)
+      # New identity format is "{hostname}:{pid}:{randomhex}
+      # Old identity format is "{hostname}:{pid}"
+      # Queue names may also have colons (namespaced).
+      # Expressing this in a single regex is unreadable
+
+      # Test the newer expected format first, only checking the older if necessary
+      original_queue, identity = key.scan(WORKING_QUEUE_REGEX).flatten
+      return original_queue, identity unless original_queue.nil? || identity.nil?
+
+      key.scan(LEGACY_WORKING_QUEUE_REGEX).flatten
+    end
+
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
     def clean_working_queues!
@@ -173,19 +194,16 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
-          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
+          original_queue, identity = extract_queue_and_identity(key)
 
-          next if hostname.nil? || pid.nil?
+          next if original_queue.nil? || identity.nil?
 
-          clean_working_queue!(key) if self.class.worker_dead?(hostname, pid, conn)
+          clean_working_queue!(original_queue, key) if self.class.worker_dead?(identity, conn)
         end
       end
     end
 
-    def clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
-
+    def clean_working_queue!(original_queue, working_queue)
       Sidekiq.redis do |conn|
         while job = conn.rpop(working_queue)
           preprocess_interrupted_job(job, original_queue)

data/spec/base_reliable_fetch_spec.rb

@@ -88,7 +88,7 @@ describe Sidekiq::BaseReliableFetch do
     Sidekiq.redis do |conn|
       sleep 0.2 # Give the time to heartbeat thread to make a loop
 
-      heartbeat_key = described_class.heartbeat_key(Socket.gethostname, ::Process.pid)
+      heartbeat_key = described_class.heartbeat_key(described_class.identity)
       heartbeat = conn.get(heartbeat_key)
 
       expect(heartbeat).not_to be_nil

data/spec/fetch_shared_examples.rb

@@ -7,111 +7,163 @@ shared_examples 'a Sidekiq fetcher' do
     let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
     let(:fetcher) { described_class.new(queues: queues) }
 
-    it 'retrieves the job and puts it to working queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
-
-      uow = fetcher.retrieve_work
+    it 'does not clean up orphaned jobs more than once per cleanup interval' do
+      Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
 
-      expect(working_queue_size('assigned')).to eq 1
-      expect(uow.queue_name).to eq 'assigned'
-      expect(uow.job).to eq job
-      expect(Sidekiq::Queue.new('assigned').size).to eq 0
-    end
+      expect(fetcher).to receive(:clean_working_queues!).once
 
-    it 'does not retrieve a job from foreign queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', job) }
+      threads = 10.times.map do
+        Thread.new do
+          fetcher.retrieve_work
+        end
+      end
 
-      expect(fetcher.retrieve_work).to be_nil
+      threads.map(&:join)
     end
 
-    it 'requeues jobs from dead working queue with incremented interrupted_count' do
+    it 'retrieves by order when strictly order is enabled' do
+      fetcher = described_class.new(strict: true, queues: ['first', 'second'])
+
       Sidekiq.redis do |conn|
-        conn.rpush(other_process_working_queue_name('assigned'), job)
+        conn.rpush('queue:first', ['msg3', 'msg2', 'msg1'])
+        conn.rpush('queue:second', 'msg4')
       end
 
-      expected_job = Sidekiq.load_json(job)
-      expected_job['interrupted_count'] = 1
-      expected_job = Sidekiq.dump_json(expected_job)
+      jobs = (1..4).map { fetcher.retrieve_work.job }
 
-      uow = fetcher.retrieve_work
+      expect(jobs).to eq ['msg1', 'msg2', 'msg3', 'msg4']
+    end
 
-      expect(uow.job).to eq expected_job
+    it 'does not starve any queue when queues are not strictly ordered' do
+      fetcher = described_class.new(queues: ['first', 'second'])
 
       Sidekiq.redis do |conn|
-        expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
+        conn.rpush('queue:first', (1..200).map { |i| "msg#{i}" })
+        conn.rpush('queue:second', 'this_job_should_not_stuck')
       end
+
+      jobs = (1..100).map { fetcher.retrieve_work.job }
+
+      expect(jobs).to include 'this_job_should_not_stuck'
     end
 
-    it 'ignores working queue keys in unknown formats' do
-      # Add a spurious non-numeric char segment at the end; this simulates any other
-      # incorrect form in general
-      malformed_key = "#{other_process_working_queue_name('assigned')}:X"
-      Sidekiq.redis do |conn|
-        conn.rpush(malformed_key, job)
-      end
+    shared_examples "basic queue handling" do |queue|
+      let (:fetcher) { described_class.new(queues: [queue]) }
 
-      uow = fetcher.retrieve_work
+      it 'retrieves the job and puts it to working queue' do
+        Sidekiq.redis { |conn| conn.rpush("queue:#{queue}", job) }
 
-      Sidekiq.redis do |conn|
-        expect(conn.llen(malformed_key)).to eq 1
+        uow = fetcher.retrieve_work
+
+        expect(working_queue_size(queue)).to eq 1
+        expect(uow.queue_name).to eq queue
+        expect(uow.job).to eq job
+        expect(Sidekiq::Queue.new(queue).size).to eq 0
       end
-    end
 
+      it 'does not retrieve a job from foreign queue' do
+        Sidekiq.redis { |conn| conn.rpush("'queue:#{queue}:not", job) }
+        expect(fetcher.retrieve_work).to be_nil
 
-    it 'does not requeue jobs from live working queue' do
-      working_queue = live_other_process_working_queue_name('assigned')
+        Sidekiq.redis { |conn| conn.rpush("'queue:not_#{queue}", job) }
+        expect(fetcher.retrieve_work).to be_nil
 
-      Sidekiq.redis do |conn|
-        conn.rpush(working_queue, job)
+        Sidekiq.redis { |conn| conn.rpush("'queue:random_name", job) }
+        expect(fetcher.retrieve_work).to be_nil
       end
 
-      uow = fetcher.retrieve_work
+      it 'requeues jobs from legacy dead working queue with incremented interrupted_count' do
+        Sidekiq.redis do |conn|
+          conn.rpush(legacy_other_process_working_queue_name(queue), job)
+        end
 
-      expect(uow).to be_nil
+        expected_job = Sidekiq.load_json(job)
+        expected_job['interrupted_count'] = 1
+        expected_job = Sidekiq.dump_json(expected_job)
 
-      Sidekiq.redis do |conn|
-        expect(conn.llen(working_queue)).to eq 1
+        uow = fetcher.retrieve_work
+
+        expect(uow).to_not be_nil
+        expect(uow.job).to eq expected_job
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(legacy_other_process_working_queue_name(queue))).to eq 0
+        end
       end
-    end
 
-    it 'does not clean up orphaned jobs more than once per cleanup interval' do
-      Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
+      it 'ignores working queue keys in unknown formats' do
+        # Add a spurious non-numeric char segment at the end; this simulates any other
+        # incorrect form in general
+        malformed_key = "#{other_process_working_queue_name(queue)}:X"
+        Sidekiq.redis do |conn|
+          conn.rpush(malformed_key, job)
+        end
 
-      expect(fetcher).to receive(:clean_working_queues!).once
+        uow = fetcher.retrieve_work
 
-      threads = 10.times.map do
-        Thread.new do
-          fetcher.retrieve_work
+        Sidekiq.redis do |conn|
+          expect(conn.llen(malformed_key)).to eq 1
         end
       end
 
-      threads.map(&:join)
-    end
+      it 'requeues jobs from dead working queue with incremented interrupted_count' do
+        Sidekiq.redis do |conn|
+          conn.rpush(other_process_working_queue_name(queue), job)
+        end
 
-    it 'retrieves by order when strictly order is enabled' do
-      fetcher = described_class.new(strict: true, queues: ['first', 'second'])
+        expected_job = Sidekiq.load_json(job)
+        expected_job['interrupted_count'] = 1
+        expected_job = Sidekiq.dump_json(expected_job)
 
-      Sidekiq.redis do |conn|
-        conn.rpush('queue:first', ['msg3', 'msg2', 'msg1'])
-        conn.rpush('queue:second', 'msg4')
+        uow = fetcher.retrieve_work
+
+        expect(uow).to_not be_nil
+        expect(uow.job).to eq expected_job
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(other_process_working_queue_name(queue))).to eq 0
+        end
       end
 
-      jobs = (1..4).map { fetcher.retrieve_work.job }
+      it 'does not requeue jobs from live working queue' do
+        working_queue = live_other_process_working_queue_name(queue)
 
-      expect(jobs).to eq ['msg1', 'msg2', 'msg3', 'msg4']
-    end
+        Sidekiq.redis do |conn|
+          conn.rpush(working_queue, job)
+        end
 
-    it 'does not starve any queue when queues are not strictly ordered' do
-      fetcher = described_class.new(queues: ['first', 'second'])
+        uow = fetcher.retrieve_work
 
-      Sidekiq.redis do |conn|
-        conn.rpush('queue:first', (1..200).map { |i| "msg#{i}" })
-        conn.rpush('queue:second', 'this_job_should_not_stuck')
+        expect(uow).to be_nil
+
+        Sidekiq.redis do |conn|
+          expect(conn.llen(working_queue)).to eq 1
+        end
       end
+    end
 
-      jobs = (1..100).map { fetcher.retrieve_work.job }
+    context 'with various queues' do
+      %w[assigned namespace:assigned namespace:deeper:assigned].each do |queue|
+        it_behaves_like "basic queue handling", queue
+      end
+    end
 
-      expect(jobs).to include 'this_job_should_not_stuck'
+    context 'with short cleanup interval' do
+      let(:short_interval) { 1 }
+      let(:fetcher) { described_class.new(queues: queues, lease_interval: short_interval, cleanup_interval: short_interval) }
+
+      it 'requeues when there is no heartbeat' do
+        Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
+        # Use of retrieve_work twice with a sleep ensures we have exercised the
+        # `identity` method to create the working queue key name and that it
+        # matches the patterns used in the cleanup
+        uow = fetcher.retrieve_work
+        sleep(short_interval + 1)
+        uow = fetcher.retrieve_work
+
+        # Will only receive a UnitOfWork if the job was detected as failed and requeued
+        expect(uow).to_not be_nil
+      end
     end
   end
 end
@@ -122,17 +174,22 @@ def working_queue_size(queue_name)
   end
 end
 
-def other_process_working_queue_name(queue)
+def legacy_other_process_working_queue_name(queue)
   "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
 end
 
+def other_process_working_queue_name(queue)
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}:#{::SecureRandom.hex(6)}"
+end
+
 def live_other_process_working_queue_name(queue)
   pid = ::Process.pid + 1
   hostname = Socket.gethostname
+  nonce = SecureRandom.hex(6)
 
   Sidekiq.redis do |conn|
-    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key(hostname, pid), 1)
+    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key("#{hostname}-#{pid}-#{nonce}"), 1)
   end
 
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}"
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}:#{nonce}"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - TEA
 - GitLab
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-18 00:00:00.000000000 Z
+date: 2021-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -63,7 +63,7 @@ homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -79,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq
 test_files: []