gitlab-sidekiq-fetcher 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a179330459058ca56cd918066793895684d63bbfcfed73eccd8687d5ed22ea15
-  data.tar.gz: fb0115218a6c35a6349f66c79a0a9eacbf1434a7fc6015100ff2dffb0bf8cef7
+  metadata.gz: 477650a08755f00beb453c4867a270fa8469a83a372b5b09ba1c030885899699
+  data.tar.gz: da3fcf3f0b67dd3f71c73ea80d09666e82fa9360cdf5cbf6c8b8e5668b85bfb5
 SHA512:
-  metadata.gz: c4ef091732aa1cd0b69b889d7649c6f4ade001989360cdb717296cbfec6db7551e379b278b64fcb823b0cf293245b72293fbef8ad8b009158060983ecf421328
-  data.tar.gz: 932751a38d73e1f1d0d147fb654ce802bc3a0f9b20319ab76ef9525fd29661ac70b505aa47e71932f9e02d6651785f8779dfdbd3723226b1f53d55d7e1e3880d
+  metadata.gz: b4d03a71a6c00e2fa55affd4af5633c2c1ef58093c8bd82b95d4a1603523d940e35b8db4643becf20236224ef065fc879b70585933569efa893e0e05481bffb9
+  data.tar.gz: 4fac10a26e3507c70927ad8689c2d8903d452cc9ed25a6522dc5c2c7e5e7fcb13d8412013eadb435a9a2feecab51130ad516553c4bbb49c72b2b36827f75c3cf

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.5.3'
+  s.version       = '0.5.4'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -68,20 +68,24 @@ module Sidekiq
       end
     end
 
-    def self.pid
-      @pid ||= ::Process.pid
+    def self.hostname
+      Socket.gethostname
     end
 
-    def self.hostname
-      @hostname ||= Socket.gethostname
+    def self.process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def self.identity
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
     def self.heartbeat
       Sidekiq.redis do |conn|
-        conn.set(heartbeat_key(hostname, pid), 1, ex: HEARTBEAT_LIFESPAN)
+        conn.set(heartbeat_key(identity), 1, ex: HEARTBEAT_LIFESPAN)
       end
 
-      Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
+      Sidekiq.logger.debug("Heartbeat for #{identity}")
     end
 
     def self.bulk_requeue(inprogress, _options)
@@ -100,9 +104,7 @@ module Sidekiq
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
     end
 
-    def self.clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
-
+    def self.clean_working_queue!(original_queue, working_queue)
       Sidekiq.redis do |conn|
         while job = conn.rpop(working_queue)
           preprocess_interrupted_job(job, original_queue)
@@ -121,6 +123,14 @@ module Sidekiq
       end
     end
 
+    def self.valid_identity_format?(identity)
+      # New format is "{hostname}:{pid}:{randomhex}
+      # Old format is "{hostname}:{pid}"
+
+      # Test the newer format first, only checking the older if necessary
+      identity.match(/[^:]*:[0-9]*:[0-9a-f]*\z/) || identity.match(/([^:]*):([0-9]*)\z/)
+    end
+
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
     def self.clean_working_queues!
@@ -128,26 +138,25 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
-          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
+          original_queue, identity = key.scan(/#{WORKING_QUEUE_PREFIX}:(queue:[^:]*):(.*)\z/).flatten
 
-          next if hostname.nil? || pid.nil?
+          next unless valid_identity_format?(identity)
 
-          clean_working_queue!(key) if worker_dead?(hostname, pid, conn)
+          clean_working_queue!(original_queue, key) if worker_dead?(identity, conn)
         end
       end
     end
 
-    def self.worker_dead?(hostname, pid, conn)
-      !conn.get(heartbeat_key(hostname, pid))
+    def self.worker_dead?(identity, conn)
+      !conn.get(heartbeat_key(identity))
     end
 
-    def self.heartbeat_key(hostname, pid)
-      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
+    def self.heartbeat_key(identity)
+      "reliable-fetcher-heartbeat-#{identity.gsub(':', '-')}"
     end
 
     def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{identity}"
     end
 
     def self.interruption_exhausted?(msg)

data/spec/base_reliable_fetch_spec.rb

@@ -87,7 +87,7 @@ describe Sidekiq::BaseReliableFetch do
     Sidekiq.redis do |conn|
       sleep 0.2 # Give the time to heartbeat thread to make a loop
 
-      heartbeat_key = described_class.heartbeat_key(Socket.gethostname, ::Process.pid)
+      heartbeat_key = described_class.heartbeat_key(described_class.identity)
       heartbeat = conn.get(heartbeat_key)
 
       expect(heartbeat).not_to be_nil

data/spec/fetch_shared_examples.rb

@@ -35,6 +35,7 @@ shared_examples 'a Sidekiq fetcher' do
 
       uow = fetcher.retrieve_work
 
+      expect(uow).to_not be_nil
       expect(uow.job).to eq expected_job
 
       Sidekiq.redis do |conn|
@@ -57,6 +58,24 @@ shared_examples 'a Sidekiq fetcher' do
       end
     end
 
+    it 'requeues jobs from legacy dead working queue with incremented interrupted_count' do
+      Sidekiq.redis do |conn|
+        conn.rpush(legacy_other_process_working_queue_name('assigned'), job)
+      end
+
+      expected_job = Sidekiq.load_json(job)
+      expected_job['interrupted_count'] = 1
+      expected_job = Sidekiq.dump_json(expected_job)
+
+      uow = fetcher.retrieve_work
+
+      expect(uow).to_not be_nil
+      expect(uow.job).to eq expected_job
+
+      Sidekiq.redis do |conn|
+        expect(conn.llen(legacy_other_process_working_queue_name('assigned'))).to eq 0
+      end
+    end
 
     it 'does not requeue jobs from live working queue' do
       working_queue = live_other_process_working_queue_name('assigned')
@@ -113,6 +132,24 @@ shared_examples 'a Sidekiq fetcher' do
 
       expect(jobs).to include 'this_job_should_not_stuck'
     end
+
+    context 'with short cleanup interval' do
+      let(:short_interval) { 1 }
+      let(:fetcher) { described_class.new(queues: queues, lease_interval: short_interval, cleanup_interval: short_interval) }
+
+      it 'requeues when there is no heartbeat' do
+        Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
+        # Use of retrieve_work twice with a sleep ensures we have exercised the
+        # `identity` method to create the working queue key name and that it
+        # matches the patterns used in the cleanup
+        uow = fetcher.retrieve_work
+        sleep(short_interval + 1)
+        uow = fetcher.retrieve_work
+
+        # Will only receive a UnitOfWork if the job was detected as failed and requeued
+        expect(uow).to_not be_nil
+      end
+    end
   end
 end
 
@@ -122,17 +159,23 @@ def working_queue_size(queue_name)
   end
 end
 
-def other_process_working_queue_name(queue)
+def legacy_other_process_working_queue_name(queue)
   "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
 end
 
+
+def other_process_working_queue_name(queue)
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}:#{::SecureRandom.hex(6)}"
+end
+
 def live_other_process_working_queue_name(queue)
   pid = ::Process.pid + 1
   hostname = Socket.gethostname
+  nonce = SecureRandom.hex(6)
 
   Sidekiq.redis do |conn|
-    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key(hostname, pid), 1)
+    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key("#{hostname}-#{pid}-#{nonce}"), 1)
   end
 
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}"
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}:#{nonce}"
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - TEA
 - GitLab
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-18 00:00:00.000000000 Z
+date: 2021-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -63,7 +63,7 @@ homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -79,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq
 test_files: []