gitlab-sidekiq-fetcher 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a71699d717aeb95cb406ed566e394a34e7978df8289db2fe89aff82046dd8e19
-  data.tar.gz: 4390a2a95507b8a6c5c08a9d3dac384f58f0d11311afd1da5973e4c61240120a
+  metadata.gz: 6b9cf610a1915d63141331ba0e4820306235fc1c58e37e0124a9700d19005b99
+  data.tar.gz: 3690c4aaff9d47c8ec108ff264984343bc3a4412123ad56ac727cdf1ae3e8fd3
 SHA512:
-  metadata.gz: fa95de2f9b33f01b45c547b23bfd4728ab88ed304ecbeaa2416e03b1a6d6bff678f097b37ae85679ef69f520a37a6f7115207170284229df706172f60d103b19
-  data.tar.gz: d7981fd4afe0abf8454caffac4c1fa8e290dd8617ce28edf45b4a29a9deb9b3a9cd583f64038cc3808b814923ab74fc370d65d9a42257b4af9fa97ea48542959
+  metadata.gz: 6d5d61280c6db3b91c8107fca593fa12246db18c727389ef31cf8edf58e923bc78a2cbbe2be505962664ed56c8189513dc0ad0312e545988b70666839f191f66
+  data.tar.gz: cd1459179a3f97b3b3194a21e8335367e9021db590cb9441613d6d88030b1b86cc495b00e1ff37dc0cf7ea683d40e6b10d3c511c7d956a70e8c26c67a4bc0903

data/.gitlab-ci.yml

@@ -40,7 +40,6 @@ integration_reliable:
   variables:
     JOB_FETCHER: reliable
 
-
 integration_basic:
   extends: .integration
   allow_failure: yes
@@ -63,7 +62,6 @@ term_interruption:
   services:
     - redis:alpine
 
-
 # rubocop:
 #   script:
 #     - bundle exec rubocop

data/Gemfile

@@ -7,6 +7,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 group :test do
   gem "rspec", '~> 3'
   gem "pry"
-  gem "sidekiq", '~> 6.0.0'
+  gem "sidekiq", '~> 6.1'
   gem 'simplecov', require: false
 end

data/Gemfile.lock

@@ -11,8 +11,6 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     rack (2.2.3)
-    rack-protection (2.0.8.1)
-      rack
     redis (4.2.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
@@ -27,11 +25,10 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    sidekiq (6.0.7)
+    sidekiq (6.1.0)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
-      rack-protection (>= 2.0.0)
-      redis (>= 4.1.0)
+      redis (>= 4.2.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -44,7 +41,7 @@ PLATFORMS
 DEPENDENCIES
   pry
   rspec (~> 3)
-  sidekiq (~> 6.0.0)
+  sidekiq (~> 6.1)
   simplecov
 
 BUNDLED WITH

data/README.md

@@ -6,6 +6,8 @@ fetches from Redis.
 
 It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
 
+**IMPORTANT NOTE:** Since version `0.7.0` this gem works only with `sidekiq >= 6.1` (which introduced Fetch API breaking changes). Please use version `~> 0.5` if you use older version of the `sidekiq` .
+
 There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
 semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
 However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.6.1'
+  s.version       = '0.7.0'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'
@@ -10,5 +10,5 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.files         = `git ls-files`.split($\)
   s.test_files    = []
-  s.add_dependency 'sidekiq', '>= 5', '< 6.1'
+  s.add_dependency 'sidekiq', '~> 6.1'
 end

data/lib/sidekiq/base_reliable_fetch.rb

@@ -41,11 +41,13 @@ module Sidekiq
     end
 
     def self.setup_reliable_fetch!(config)
-      config.options[:fetch] = if config.options[:semi_reliable_fetch]
-                                 Sidekiq::SemiReliableFetch
-                               else
-                                 Sidekiq::ReliableFetch
-                               end
+      fetch_strategy = if config.options[:semi_reliable_fetch]
+                         Sidekiq::SemiReliableFetch
+                       else
+                         Sidekiq::ReliableFetch
+                       end
+
+      config.options[:fetch] = fetch_strategy.new(config.options)
 
       Sidekiq.logger.info('GitLab reliable fetch activated!')
 
@@ -84,7 +86,44 @@ module Sidekiq
       Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
     end
 
-    def self.bulk_requeue(inprogress, _options)
+    def self.worker_dead?(hostname, pid, conn)
+      !conn.get(heartbeat_key(hostname, pid))
+    end
+
+    def self.heartbeat_key(hostname, pid)
+      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
+    end
+
+    def self.working_queue_name(queue)
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+    end
+
+    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
+                :queues, :use_semi_reliable_fetch,
+                :strictly_ordered_queues
+
+    def initialize(options)
+      raise ArgumentError, 'missing queue list' unless options[:queues]
+
+      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
+      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
+      @last_try_to_take_lease_at = 0
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues].map { |q| "queue:#{q}" }
+    end
+
+    def retrieve_work
+      clean_working_queues! if take_lease
+
+      retrieve_unit_of_work
+    end
+
+    def retrieve_unit_of_work
+      raise NotImplementedError,
+            "#{self.class} does not implement #{__method__}"
+    end
+
+    def bulk_requeue(inprogress, _options)
       return if inprogress.empty?
 
       Sidekiq.redis do |conn|
@@ -92,7 +131,7 @@ module Sidekiq
           conn.multi do |multi|
             preprocess_interrupted_job(unit_of_work.job, unit_of_work.queue, multi)
 
-            multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
+            multi.lrem(self.class.working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
           end
         end
       end
@@ -100,17 +139,9 @@ module Sidekiq
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
     end
 
-    def self.clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
-
-      Sidekiq.redis do |conn|
-        while job = conn.rpop(working_queue)
-          preprocess_interrupted_job(job, original_queue)
-        end
-      end
-    end
+    private
 
-    def self.preprocess_interrupted_job(job, queue, conn = nil)
+    def preprocess_interrupted_job(job, queue, conn = nil)
       msg = Sidekiq.load_json(job)
       msg['interrupted_count'] = msg['interrupted_count'].to_i + 1
 
@@ -121,9 +152,23 @@ module Sidekiq
       end
     end
 
+    # If you want this method to be run in a scope of multi connection
+    # you need to pass it
+    def requeue_job(queue, msg, conn)
+      with_connection(conn) do |conn|
+        conn.lpush(queue, Sidekiq.dump_json(msg))
+      end
+
+      Sidekiq.logger.info(
+        message: "Pushed job #{msg['jid']} back to queue #{queue}",
+        jid: msg['jid'],
+        queue: queue
+      )
+    end
+
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
-    def self.clean_working_queues!
+    def clean_working_queues!
       Sidekiq.logger.info('Cleaning working queues')
 
       Sidekiq.redis do |conn|
@@ -133,30 +178,28 @@ module Sidekiq
 
           continue if hostname.nil? || pid.nil?
 
-          clean_working_queue!(key) if worker_dead?(hostname, pid, conn)
+          clean_working_queue!(key) if self.class.worker_dead?(hostname, pid, conn)
         end
       end
     end
 
-    def self.worker_dead?(hostname, pid, conn)
-      !conn.get(heartbeat_key(hostname, pid))
-    end
-
-    def self.heartbeat_key(hostname, pid)
-      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
-    end
+    def clean_working_queue!(working_queue)
+      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
 
-    def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+      Sidekiq.redis do |conn|
+        while job = conn.rpop(working_queue)
+          preprocess_interrupted_job(job, original_queue)
+        end
+      end
     end
 
-    def self.interruption_exhausted?(msg)
+    def interruption_exhausted?(msg)
       return false if max_retries_after_interruption(msg['class']) < 0
 
       msg['interrupted_count'].to_i >= max_retries_after_interruption(msg['class'])
     end
 
-    def self.max_retries_after_interruption(worker_class)
+    def max_retries_after_interruption(worker_class)
       max_retries_after_interruption = nil
 
       max_retries_after_interruption ||= begin
@@ -169,7 +212,7 @@ module Sidekiq
       max_retries_after_interruption
     end
 
-    def self.send_to_quarantine(msg, multi_connection = nil)
+    def send_to_quarantine(msg, multi_connection = nil)
       Sidekiq.logger.warn(
         class: msg['class'],
         jid: msg['jid'],
@@ -180,52 +223,13 @@ module Sidekiq
       Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
     end
 
-    # If you want this method to be run is a scope of multi connection
-    # you need to pass it
-    def self.requeue_job(queue, msg, conn)
-      with_connection(conn) do |conn|
-        conn.lpush(queue, Sidekiq.dump_json(msg))
-      end
-
-      Sidekiq.logger.info(
-        message: "Pushed job #{msg['jid']} back to queue #{queue}",
-        jid: msg['jid'],
-        queue: queue
-      )
-    end
-
     # Yield block with an existing connection or creates another one
-    def self.with_connection(conn, &block)
+    def with_connection(conn)
       return yield(conn) if conn
 
-      Sidekiq.redis { |conn| yield(conn) }
-    end
-
-    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
-                :queues, :use_semi_reliable_fetch,
-                :strictly_ordered_queues
-
-    def initialize(options)
-      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
-      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
-      @last_try_to_take_lease_at = 0
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
-    end
-
-    def retrieve_work
-      self.class.clean_working_queues! if take_lease
-
-      retrieve_unit_of_work
+      Sidekiq.redis { |redis_conn| yield(redis_conn) }
     end
 
-    def retrieve_unit_of_work
-      raise NotImplementedError,
-        "#{self.class} does not implement #{__method__}"
-    end
-
-    private
-
     def take_lease
       return unless allowed_to_take_a_lease?
 

data/lib/sidekiq/reliable_fetch.rb

@@ -6,23 +6,21 @@ module Sidekiq
     # we inject a regular sleep into the loop.
     RELIABLE_FETCH_IDLE_TIMEOUT = 5 # seconds
 
-    attr_reader :queues_iterator, :queues_size
+    attr_reader :queues_size
 
     def initialize(options)
       super
 
+      @queues = queues.uniq if strictly_ordered_queues
       @queues_size = queues.size
-      @queues_iterator = queues.cycle
     end
 
     private
 
     def retrieve_unit_of_work
-      @queues_iterator.rewind if strictly_ordered_queues
-
-      queues_size.times do
-        queue = queues_iterator.next
+      queues_list = strictly_ordered_queues ? queues : queues.shuffle
 
+      queues_list.each do |queue|
         work = Sidekiq.redis do |conn|
           conn.rpoplpush(queue, self.class.working_queue_name(queue))
         end

data/spec/base_reliable_fetch_spec.rb

@@ -39,14 +39,15 @@ describe Sidekiq::BaseReliableFetch do
     end
   end
 
-  describe '.bulk_requeue' do
+  describe '#bulk_requeue' do
+    let(:options) { { queues: %w[foo bar] } }
     let!(:queue1) { Sidekiq::Queue.new('foo') }
     let!(:queue2) { Sidekiq::Queue.new('bar') }
 
     it 'requeues the bulk' do
       uow = described_class::UnitOfWork
       jobs = [ uow.new('queue:foo', job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
-      described_class.bulk_requeue(jobs, queues: [])
+      described_class.new(options).bulk_requeue(jobs, nil)
 
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
@@ -56,7 +57,7 @@ describe Sidekiq::BaseReliableFetch do
       uow = described_class::UnitOfWork
       interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
       jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
-      described_class.bulk_requeue(jobs, queues: [])
+      described_class.new(options).bulk_requeue(jobs, nil)
 
       expect(queue1.size).to eq 1
       expect(queue2.size).to eq 1
@@ -69,7 +70,7 @@ describe Sidekiq::BaseReliableFetch do
       uow = described_class::UnitOfWork
       interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
       jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
-      described_class.bulk_requeue(jobs, queues: [])
+      described_class.new(options).bulk_requeue(jobs, nil)
 
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
@@ -80,7 +81,7 @@ describe Sidekiq::BaseReliableFetch do
   end
 
   it 'sets heartbeat' do
-    config = double(:sidekiq_config, options: { queues: [] })
+    config = double(:sidekiq_config, options: { queues: %w[foo bar] })
 
     heartbeat_thread = described_class.setup_reliable_fetch!(config)
 

data/spec/fetch_shared_examples.rb

@@ -5,7 +5,7 @@ shared_examples 'a Sidekiq fetcher' do
 
   describe '#retrieve_work' do
     let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
-    let(:fetcher) { described_class.new(queues: ['assigned']) }
+    let(:fetcher) { described_class.new(queues: queues) }
 
     it 'retrieves the job and puts it to working queue' do
       Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
@@ -61,11 +61,11 @@ shared_examples 'a Sidekiq fetcher' do
     it 'does not clean up orphaned jobs more than once per cleanup interval' do
       Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
 
-      expect(described_class).to receive(:clean_working_queues!).once
+      expect(fetcher).to receive(:clean_working_queues!).once
 
       threads = 10.times.map do
         Thread.new do
-          described_class.new(queues: ['assigned']).retrieve_work
+          fetcher.retrieve_work
         end
       end
 

data/tests/reliability/reliability_test.rb

@@ -57,7 +57,7 @@ end
 def spawn_workers
   pids = []
   NUMBER_OF_WORKERS.times do
-    pids << spawn('sidekiq -r ./config.rb')
+    pids << spawn('sidekiq -q default -q low -q high -r ./config.rb')
   end
 
   pids

data/tests/support/utils.rb

@@ -11,7 +11,7 @@ def spawn_workers(number)
   pids = []
 
   number.times do
-    pids << spawn('sidekiq -r ./config.rb')
+    pids << spawn('sidekiq -q default -q high -q low -r ./config.rb')
   end
 
   pids

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - TEA
@@ -9,26 +9,20 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-03 00:00:00.000000000 Z
+date: 2020-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '6.1'
 description: Redis reliable queue pattern implemented in Sidekiq