gitlab-sidekiq-fetcher 0.5.6 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 789f9e7424fe05ba56d1a5dc4aae2bd1cc750cb4c6ab3a0d87055876dec4ac63
-  data.tar.gz: c17bca12b0e63b47c3725000de1430f69cfdef6c4b5b791e40deacc0da0e2b33
+  metadata.gz: 4815e3e75915230d7b2eaf2fe3fa0daa288ec4c670b2cd211cb659aff4838788
+  data.tar.gz: c8b491f2d1a2678ef40fe856a55bf89341e041070c40941ca693685fa3c048cf
 SHA512:
-  metadata.gz: 9684d40185a5cd89a8ada02ae8563a0ae8a278866cfe59d11e5e2cc8cbcb610f29998dc4f9df0fc07720a723f6823c39e41f375542c0d9621fde87bbfccfb01c
-  data.tar.gz: 146e99c8e8fd388d56dbbd099658dceb8210101df31ab3d362eeeefa3fd52e75e469df24e1b06be5adae2231c9c978a84f2e0681b3a0bc91b8b5f5798c941f96
+  metadata.gz: 1667fb3ffb47117ac3c756c07a85867d96871b5489e820823735b62912df0d82b4d6c10be478e1f291d637d2ed15aff99a958f4fd884df98f026ee8c8f9c4c83
+  data.tar.gz: 5dad56f30515be87e79c58c3b5dce301213d427fe579309bca2a525e2b8664b94f0763c7eabfb2443324c5d8ecf5403f9826a0772e3b09b74e215e72ebfc4226

data/Gemfile

@@ -9,5 +9,4 @@ group :test do
   gem "pry"
   gem "sidekiq", '~> 5.0'
   gem 'simplecov', require: false
-  gem 'stub_env', '~> 1.0'
 end

data/Gemfile.lock

@@ -36,8 +36,6 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    stub_env (1.0.4)
-      rspec (>= 2.0, < 4.0)
 
 PLATFORMS
   ruby
@@ -47,7 +45,6 @@ DEPENDENCIES
   rspec (~> 3)
   sidekiq (~> 5.0)
   simplecov
-  stub_env (~> 1.0)
 
 BUNDLED WITH
    1.17.1

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.5.6'
+  s.version       = '0.6.0'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'
@@ -10,5 +10,5 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.files         = `git ls-files`.split($\)
   s.test_files    = []
-  s.add_dependency 'sidekiq', '~> 5'
+  s.add_dependency 'sidekiq', '>= 5', '< 7'
 end

data/lib/sidekiq/base_reliable_fetch.rb

@@ -21,10 +21,6 @@ module Sidekiq
     # How much time a job can be interrupted
     DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION = 3
 
-    # Regexes for matching working queue keys
-    WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*:[0-9a-f]*)\z/.freeze
-    LEGACY_WORKING_QUEUE_REGEX = /#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*)\z/.freeze
-
     UnitOfWork = Struct.new(:queue, :job) do
       def acknowledge
         Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
@@ -45,12 +41,10 @@ module Sidekiq
     end
 
     def self.setup_reliable_fetch!(config)
-      config.options[:fetch] = if config.options[:semi_reliable_fetch]
-                                 Sidekiq::SemiReliableFetch
-                               else
-                                 Sidekiq::ReliableFetch
-                               end
+      fetch = config.options[:semi_reliable_fetch] ? SemiReliableFetch : ReliableFetch
+      fetch = fetch.new(config.options) if Sidekiq::VERSION >= '6'
 
+      config.options[:fetch] = fetch
       Sidekiq.logger.info('GitLab reliable fetch activated!')
 
       start_heartbeat_thread
@@ -72,24 +66,24 @@ module Sidekiq
       end
     end
 
-    def self.hostname
-      Socket.gethostname
-    end
-
-    def self.process_nonce
-      @@process_nonce ||= SecureRandom.hex(6)
+    def self.pid
+      @pid ||= ::Process.pid
     end
 
-    def self.identity
-      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
+    def self.hostname
+      @hostname ||= Socket.gethostname
     end
 
     def self.heartbeat
       Sidekiq.redis do |conn|
-        conn.set(heartbeat_key(identity), 1, ex: HEARTBEAT_LIFESPAN)
+        conn.set(heartbeat_key(hostname, pid), 1, ex: HEARTBEAT_LIFESPAN)
       end
 
-      Sidekiq.logger.debug("Heartbeat for #{identity}")
+      Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
+    end
+
+    def bulk_requeue(inprogress, options)
+      self.class.bulk_requeue(inprogress, options)
     end
 
     def self.bulk_requeue(inprogress, _options)
@@ -108,7 +102,9 @@ module Sidekiq
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
     end
 
-    def self.clean_working_queue!(original_queue, working_queue)
+    def self.clean_working_queue!(working_queue)
+      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
+
       Sidekiq.redis do |conn|
         while job = conn.rpop(working_queue)
           preprocess_interrupted_job(job, original_queue)
@@ -127,19 +123,6 @@ module Sidekiq
       end
     end
 
-    def self.extract_queue_and_identity(key)
-      # New identity format is "{hostname}:{pid}:{randomhex}
-      # Old identity format is "{hostname}:{pid}"
-      # Queue names may also have colons (namespaced).
-      # Expressing this in a single regex is unreadable
-
-      # Test the newer expected format first, only checking the older if necessary
-      original_queue, identity = key.scan(WORKING_QUEUE_REGEX).flatten
-      return original_queue, identity unless original_queue.nil? || identity.nil?
-
-      key.scan(LEGACY_WORKING_QUEUE_REGEX).flatten
-    end
-
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
     def self.clean_working_queues!
@@ -147,25 +130,26 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          original_queue, identity = extract_queue_and_identity(key)
+          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
+          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
 
-          next if original_queue.nil? || identity.nil?
+          continue if hostname.nil? || pid.nil?
 
-          clean_working_queue!(original_queue, key) if worker_dead?(identity, conn)
+          clean_working_queue!(key) if worker_dead?(hostname, pid, conn)
         end
       end
     end
 
-    def self.worker_dead?(identity, conn)
-      !conn.get(heartbeat_key(identity))
+    def self.worker_dead?(hostname, pid, conn)
+      !conn.get(heartbeat_key(hostname, pid))
     end
 
-    def self.heartbeat_key(identity)
-      "reliable-fetcher-heartbeat-#{identity.gsub(':', '-')}"
+    def self.heartbeat_key(hostname, pid)
+      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
     end
 
     def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{identity}"
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
     end
 
     def self.interruption_exhausted?(msg)

data/lib/sidekiq/semi_reliable_fetch.rb

@@ -5,14 +5,14 @@ module Sidekiq
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down. This constant is only used
     # for semi-reliable fetch.
-    DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
+    SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
 
     def initialize(options)
       super
 
       if strictly_ordered_queues
         @queues = @queues.uniq
-        @queues << semi_reliable_fetch_timeout
+        @queues << SEMI_RELIABLE_FETCH_TIMEOUT
       end
     end
 
@@ -36,13 +36,9 @@ module Sidekiq
         @queues
       else
         queues = @queues.shuffle.uniq
-        queues << semi_reliable_fetch_timeout
+        queues << SEMI_RELIABLE_FETCH_TIMEOUT
         queues
       end
     end
-
-    def semi_reliable_fetch_timeout
-      @semi_reliable_fetch_timeout ||= ENV['SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT']&.to_i || DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT
-    end
   end
 end

data/spec/base_reliable_fetch_spec.rb

@@ -80,14 +80,14 @@ describe Sidekiq::BaseReliableFetch do
   end
 
   it 'sets heartbeat' do
-    config = double(:sidekiq_config, options: {})
+    config = double(:sidekiq_config, options: { queues: [] })
 
     heartbeat_thread = described_class.setup_reliable_fetch!(config)
 
     Sidekiq.redis do |conn|
       sleep 0.2 # Give the time to heartbeat thread to make a loop
 
-      heartbeat_key = described_class.heartbeat_key(described_class.identity)
+      heartbeat_key = described_class.heartbeat_key(Socket.gethostname, ::Process.pid)
       heartbeat = conn.get(heartbeat_key)
 
       expect(heartbeat).not_to be_nil

data/spec/fetch_shared_examples.rb

@@ -35,7 +35,6 @@ shared_examples 'a Sidekiq fetcher' do
 
       uow = fetcher.retrieve_work
 
-      expect(uow).to_not be_nil
       expect(uow.job).to eq expected_job
 
       Sidekiq.redis do |conn|
@@ -43,40 +42,6 @@ shared_examples 'a Sidekiq fetcher' do
       end
     end
 
-    it 'ignores working queue keys in unknown formats' do
-      # Add a spurious non-numeric char segment at the end; this simulates any other
-      # incorrect form in general
-      malformed_key = "#{other_process_working_queue_name('assigned')}:X"
-      Sidekiq.redis do |conn|
-        conn.rpush(malformed_key, job)
-      end
-
-      uow = fetcher.retrieve_work
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(malformed_key)).to eq 1
-      end
-    end
-
-    it 'requeues jobs from legacy dead working queue with incremented interrupted_count' do
-      Sidekiq.redis do |conn|
-        conn.rpush(legacy_other_process_working_queue_name('assigned'), job)
-      end
-
-      expected_job = Sidekiq.load_json(job)
-      expected_job['interrupted_count'] = 1
-      expected_job = Sidekiq.dump_json(expected_job)
-
-      uow = fetcher.retrieve_work
-
-      expect(uow).to_not be_nil
-      expect(uow.job).to eq expected_job
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(legacy_other_process_working_queue_name('assigned'))).to eq 0
-      end
-    end
-
     it 'does not requeue jobs from live working queue' do
       working_queue = live_other_process_working_queue_name('assigned')
 
@@ -132,104 +97,6 @@ shared_examples 'a Sidekiq fetcher' do
 
       expect(jobs).to include 'this_job_should_not_stuck'
     end
-
-    context 'with namespaced queues' do
-      let (:queue) { 'namespace:assigned' }
-      let (:fetcher) { described_class.new(queues: [queue]) }
-
-      it 'requeues jobs from dead namespaced working queue with incremented interrupted_count' do
-        Sidekiq.redis do |conn|
-          conn.rpush(other_process_working_queue_name(queue), job)
-        end
-
-        expected_job = Sidekiq.load_json(job)
-        expected_job['interrupted_count'] = 1
-        expected_job = Sidekiq.dump_json(expected_job)
-
-        uow = fetcher.retrieve_work
-
-        expect(uow).to_not be_nil
-        expect(uow.job).to eq expected_job
-
-        Sidekiq.redis do |conn|
-          expect(conn.llen(other_process_working_queue_name(queue))).to eq 0
-        end
-      end
-
-      it 'does not requeue jobs in a namespaced queue from live working queue' do
-        working_queue = live_other_process_working_queue_name(queue)
-
-        Sidekiq.redis do |conn|
-          conn.rpush(working_queue, job)
-        end
-
-        uow = fetcher.retrieve_work
-
-        expect(uow).to be_nil
-
-        Sidekiq.redis do |conn|
-          expect(conn.llen(working_queue)).to eq 1
-        end
-      end
-    end
-
-    context 'with deeper namespaced queues' do
-      let (:queue) { 'deep:namespace:assigned' }
-      let (:fetcher) { described_class.new(queues: [queue]) }
-
-      it 'requeues jobs from dead namespaced working queue with incremented interrupted_count' do
-        Sidekiq.redis do |conn|
-          conn.rpush(other_process_working_queue_name(queue), job)
-        end
-
-        expected_job = Sidekiq.load_json(job)
-        expected_job['interrupted_count'] = 1
-        expected_job = Sidekiq.dump_json(expected_job)
-
-        uow = fetcher.retrieve_work
-
-        expect(uow).to_not be_nil
-        expect(uow.job).to eq expected_job
-
-        Sidekiq.redis do |conn|
-          expect(conn.llen(other_process_working_queue_name(queue))).to eq 0
-        end
-      end
-
-      it 'does not requeue jobs in a deeper namespaced queue from live working queue' do
-        working_queue = live_other_process_working_queue_name(queue)
-
-        Sidekiq.redis do |conn|
-          conn.rpush(working_queue, job)
-        end
-
-        uow = fetcher.retrieve_work
-
-        expect(uow).to be_nil
-
-        Sidekiq.redis do |conn|
-          expect(conn.llen(working_queue)).to eq 1
-        end
-      end
-    end
-
-    context 'with short cleanup interval' do
-      let(:short_interval) { 1 }
-      let(:fetcher) { described_class.new(queues: queues, lease_interval: short_interval, cleanup_interval: short_interval) }
-
-      it 'requeues when there is no heartbeat' do
-        Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
-        # Use of retrieve_work twice with a sleep ensures we have exercised the
-        # `identity` method to create the working queue key name and that it
-        # matches the patterns used in the cleanup
-        uow = fetcher.retrieve_work
-        sleep(short_interval + 1)
-        uow = fetcher.retrieve_work
-
-        # Will only receive a UnitOfWork if the job was detected as failed and requeued
-        expect(uow).to_not be_nil
-      end
-    end
   end
 end
 
@@ -239,23 +106,17 @@ def working_queue_size(queue_name)
   end
 end
 
-def legacy_other_process_working_queue_name(queue)
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
-end
-
-
 def other_process_working_queue_name(queue)
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}:#{::SecureRandom.hex(6)}"
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
 end
 
 def live_other_process_working_queue_name(queue)
   pid = ::Process.pid + 1
   hostname = Socket.gethostname
-  nonce = SecureRandom.hex(6)
 
   Sidekiq.redis do |conn|
-    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key("#{hostname}-#{pid}-#{nonce}"), 1)
+    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key(hostname, pid), 1)
   end
 
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}:#{nonce}"
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}"
 end

data/spec/semi_reliable_fetch_spec.rb

@@ -5,39 +5,4 @@ require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::SemiReliableFetch do
   include_examples 'a Sidekiq fetcher'
-
-  describe '#retrieve_work' do
-    context 'timeout config' do
-      let(:queues) { ['stuff_to_do'] }
-      let(:fetcher) { described_class.new(queues: queues) }
-
-      before do
-        stub_env('SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT', timeout)
-      end
-
-      context 'when the timeout is not configured' do
-        let(:timeout) { nil }
-
-        it 'brpops with the default timeout timeout' do
-          Sidekiq.redis do |connection|
-            expect(connection).to receive(:brpop).with("queue:stuff_to_do", 2).once.and_call_original
-
-            fetcher.retrieve_work
-          end
-        end
-      end
-
-      context 'when the timeout is set in the env' do
-        let(:timeout) { '5' }
-
-        it 'brpops with the default timeout timeout' do
-          Sidekiq.redis do |connection|
-            expect(connection).to receive(:brpop).with("queue:stuff_to_do", 5).once.and_call_original
-
-            fetcher.retrieve_work
-          end
-        end
-      end
-    end
-  end
 end

data/spec/spec_helper.rb

@@ -3,7 +3,6 @@ require 'sidekiq/util'
 require 'sidekiq/api'
 require 'pry'
 require 'simplecov'
-require 'stub_env'
 
 SimpleCov.start
 
@@ -30,7 +29,6 @@ Sidekiq.logger.level = Logger::ERROR
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|
-  config.include StubEnv::Helpers
   # rspec-expectations config goes here. You can use an alternate
   # assertion/expectation library such as wrong or the stdlib/minitest
   # assertions if you prefer.

metadata

@@ -1,30 +1,36 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.6.0
 platform: ruby
 authors:
 - TEA
 - GitLab
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-24 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 description: Redis reliable queue pattern implemented in Sidekiq
 email: valery@gitlab.com
 executables: []
@@ -63,7 +69,7 @@ homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -79,7 +85,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq
 test_files: []