gitlab-sidekiq-fetcher 0.4.0 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc2e47cf7679deb6a6d526e199a09c20a671e3b55bad22d7c05ce17405eb6103
-  data.tar.gz: e71949587df8a635223ca8fa36339949df771f493e7edda0a4d9c34198600fb5
+  metadata.gz: a179330459058ca56cd918066793895684d63bbfcfed73eccd8687d5ed22ea15
+  data.tar.gz: fb0115218a6c35a6349f66c79a0a9eacbf1434a7fc6015100ff2dffb0bf8cef7
 SHA512:
-  metadata.gz: ea7d6b7283354053a4f9fc24f419ab56da97efff3ce3ac3a2a10517ab2a1bd184a44b0ffb9d39b1dbdf27b23032c58b1139a18d8f2ec1bf5357deed65187e3ed
-  data.tar.gz: ae4c78eca271dc63abf98bc112a582a2f56fc4e1e7be843189c30193ff0dabd99232f5a019b9b69d45cd1f08ca7bd88afe0c4341e0256c88d352441474a08400
+  metadata.gz: c4ef091732aa1cd0b69b889d7649c6f4ade001989360cdb717296cbfec6db7551e379b278b64fcb823b0cf293245b72293fbef8ad8b009158060983ecf421328
+  data.tar.gz: 932751a38d73e1f1d0d147fb654ce802bc3a0f9b20319ab76ef9525fd29661ac70b505aa47e71932f9e02d6651785f8779dfdbd3723226b1f53d55d7e1e3880d

data/.gitignore

@@ -1,2 +1,3 @@
 *.gem
 coverage
+.DS_Store

data/.gitlab-ci.yml

@@ -3,7 +3,7 @@ image: "ruby:2.5"
 before_script:
   - ruby -v
   - which ruby
-  - gem install bundler --no-ri --no-rdoc
+  - gem install bundler
   - bundle install --jobs $(nproc)  "${FLAGS[@]}"
 
 variables:
@@ -25,7 +25,7 @@ rspec:
 .integration:
   stage: test
   script:
-    - cd test
+    - cd tests/reliability
     - bundle exec ruby reliability_test.rb
   services:
     - redis:alpine
@@ -47,6 +47,22 @@ integration_basic:
   variables:
     JOB_FETCHER: basic
 
+kill_interruption:
+  stage: test
+  script:
+    - cd tests/interruption
+    - bundle exec ruby test_kill_signal.rb
+  services:
+    - redis:alpine
+
+term_interruption:
+  stage: test
+  script:
+    - cd tests/interruption
+    - bundle exec ruby test_term_signal.rb
+  services:
+    - redis:alpine
+
 
 # rubocop:
 #   script:

data/README.md

@@ -10,6 +10,17 @@ There are two strategies implemented: [Reliable fetch](http://redis.io/commands/
 semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
 However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.
 
+### Interruption handling
+
+Sidekiq expects any job to report succcess or to fail. In the last case, Sidekiq puts `retry_count` counter
+into the job and keeps to re-run the job until the counter reched the maximum allowed value. When the job has
+not been given a chance to finish its work(to report success or fail), for example, when it was killed forcibly or when the job was requeued, after receiving TERM signal, the standard retry mechanisme does not get into the game and the job will be retried indefinatelly. This is why Reliable fetcher maintains a special counter `interrupted_count`
+which is used to limit the amount of such retries. In both cases, Reliable Fetcher increments counter `interrupted_count` and rejects the job from running again when the counter exceeds `max_retries_after_interruption` times (default: 3 times).
+Such a job will be put to `interrupted` queue. This queue mostly behaves as Sidekiq Dead queue so it only stores a limited amount of jobs for a limited term. Same as for Dead queue, all the limits are configurable via `interrupted_max_jobs` (default: 10_000) and `interrupted_timeout_in_seconds` (default: 3 months) Sidekiq option keys.
+
+You can also disable special handling of interrupted jobs by setting `max_retries_after_interruption` into `-1`.
+In this case, interrupted jobs will be run without any limits from Reliable Fetcher and they won't be put into Interrupted queue.
+
 
 ## Installation
 

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,14 +1,14 @@
 Gem::Specification.new do |s|
-  s.name        = 'gitlab-sidekiq-fetcher'
-  s.version     = '0.4.0'
-  s.authors     = ['TEA', 'GitLab']
-  s.email       = 'valery@gitlab.com'
-  s.license     = 'LGPL-3.0'
-  s.homepage    = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
-  s.summary     = 'Reliable fetch extension for Sidekiq'
-  s.description = 'Redis reliable queue pattern implemented in Sidekiq'
+  s.name          = 'gitlab-sidekiq-fetcher'
+  s.version       = '0.5.3'
+  s.authors       = ['TEA', 'GitLab']
+  s.email         = 'valery@gitlab.com'
+  s.license       = 'LGPL-3.0'
+  s.homepage      = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
+  s.summary       = 'Reliable fetch extension for Sidekiq'
+  s.description   = 'Redis reliable queue pattern implemented in Sidekiq'
   s.require_paths = ['lib']
-  s.files = `git ls-files`.split($\)
-  s.test_files  = []
+  s.files         = `git ls-files`.split($\)
+  s.test_files    = []
   s.add_dependency 'sidekiq', '~> 5'
 end

data/lib/sidekiq-reliable-fetch.rb

@@ -1,4 +1,5 @@
 require 'sidekiq'
+require 'sidekiq/api'
 
 require_relative 'sidekiq/base_reliable_fetch'
 require_relative 'sidekiq/reliable_fetch'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'interrupted_set'
+
 module Sidekiq
   class BaseReliableFetch
     DEFAULT_CLEANUP_INTERVAL = 60 * 60  # 1 hour
@@ -16,6 +18,9 @@ module Sidekiq
     # Defines the COUNT parameter that will be passed to Redis SCAN command
     SCAN_COUNT = 1000
 
+    # How much time a job can be interrupted
+    DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION = 3
+
     UnitOfWork = Struct.new(:queue, :job) do
       def acknowledge
         Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
@@ -82,22 +87,61 @@ module Sidekiq
     def self.bulk_requeue(inprogress, _options)
       return if inprogress.empty?
 
-      Sidekiq.logger.debug('Re-queueing terminated jobs')
-
       Sidekiq.redis do |conn|
         inprogress.each do |unit_of_work|
           conn.multi do |multi|
-            multi.lpush(unit_of_work.queue, unit_of_work.job)
+            preprocess_interrupted_job(unit_of_work.job, unit_of_work.queue, multi)
+
             multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
           end
         end
       end
-
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => e
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
     end
 
+    def self.clean_working_queue!(working_queue)
+      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
+
+      Sidekiq.redis do |conn|
+        while job = conn.rpop(working_queue)
+          preprocess_interrupted_job(job, original_queue)
+        end
+      end
+    end
+
+    def self.preprocess_interrupted_job(job, queue, conn = nil)
+      msg = Sidekiq.load_json(job)
+      msg['interrupted_count'] = msg['interrupted_count'].to_i + 1
+
+      if interruption_exhausted?(msg)
+        send_to_quarantine(msg, conn)
+      else
+        requeue_job(queue, msg, conn)
+      end
+    end
+
+    # Detect "old" jobs and requeue them because the worker they were assigned
+    # to probably failed miserably.
+    def self.clean_working_queues!
+      Sidekiq.logger.info('Cleaning working queues')
+
+      Sidekiq.redis do |conn|
+        conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
+          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
+          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
+
+          next if hostname.nil? || pid.nil?
+
+          clean_working_queue!(key) if worker_dead?(hostname, pid, conn)
+        end
+      end
+    end
+
+    def self.worker_dead?(hostname, pid, conn)
+      !conn.get(heartbeat_key(hostname, pid))
+    end
+
     def self.heartbeat_key(hostname, pid)
       "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
     end
@@ -106,6 +150,57 @@ module Sidekiq
       "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
     end
 
+    def self.interruption_exhausted?(msg)
+      return false if max_retries_after_interruption(msg['class']) < 0
+
+      msg['interrupted_count'].to_i >= max_retries_after_interruption(msg['class'])
+    end
+
+    def self.max_retries_after_interruption(worker_class)
+      max_retries_after_interruption = nil
+
+      max_retries_after_interruption ||= begin
+        Object.const_get(worker_class).sidekiq_options[:max_retries_after_interruption]
+      rescue NameError
+      end
+
+      max_retries_after_interruption ||= Sidekiq.options[:max_retries_after_interruption]
+      max_retries_after_interruption ||= DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION
+      max_retries_after_interruption
+    end
+
+    def self.send_to_quarantine(msg, multi_connection = nil)
+      Sidekiq.logger.warn(
+        class: msg['class'],
+        jid: msg['jid'],
+        message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']} to interrupted queue)
+      )
+
+      job = Sidekiq.dump_json(msg)
+      Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
+    end
+
+    # If you want this method to be run is a scope of multi connection
+    # you need to pass it
+    def self.requeue_job(queue, msg, conn)
+      with_connection(conn) do |conn|
+        conn.lpush(queue, Sidekiq.dump_json(msg))
+      end
+
+      Sidekiq.logger.info(
+        message: "Pushed job #{msg['jid']} back to queue #{queue}",
+        jid: msg['jid'],
+        queue: queue
+      )
+    end
+
+    # Yield block with an existing connection or creates another one
+    def self.with_connection(conn, &block)
+      return yield(conn) if conn
+
+      Sidekiq.redis { |conn| yield(conn) }
+    end
+
     attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
                 :queues, :use_semi_reliable_fetch,
                 :strictly_ordered_queues
@@ -119,7 +214,7 @@ module Sidekiq
     end
 
     def retrieve_work
-      clean_working_queues! if take_lease
+      self.class.clean_working_queues! if take_lease
 
       retrieve_unit_of_work
     end
@@ -131,43 +226,6 @@ module Sidekiq
 
     private
 
-    def clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
-
-      Sidekiq.redis do |conn|
-        count = 0
-
-        while conn.rpoplpush(working_queue, original_queue) do
-          count += 1
-        end
-
-        Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
-      end
-    end
-
-    # Detect "old" jobs and requeue them because the worker they were assigned
-    # to probably failed miserably.
-    def clean_working_queues!
-      Sidekiq.logger.info("Cleaning working queues")
-
-      Sidekiq.redis do |conn|
-        conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
-          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
-
-          continue if hostname.nil? || pid.nil?
-
-          clean_working_queue!(key) if worker_dead?(hostname, pid)
-        end
-      end
-    end
-
-    def worker_dead?(hostname, pid)
-      Sidekiq.redis do |conn|
-        !conn.get(self.class.heartbeat_key(hostname, pid))
-      end
-    end
-
     def take_lease
       return unless allowed_to_take_a_lease?
 

data/lib/sidekiq/interrupted_set.rb

@@ -0,0 +1,47 @@
+require 'sidekiq/api'
+
+module Sidekiq
+  class InterruptedSet < ::Sidekiq::JobSet
+    DEFAULT_MAX_CAPACITY = 10_000
+    DEFAULT_MAX_TIMEOUT = 90 * 24 * 60 * 60 # 3 months
+
+    def initialize
+      super "interrupted"
+    end
+
+    def put(message, opts = {})
+      now = Time.now.to_f
+
+      with_multi_connection(opts[:connection]) do |conn|
+        conn.zadd(name, now.to_s, message)
+        conn.zremrangebyscore(name, '-inf', now - self.class.timeout)
+        conn.zremrangebyrank(name, 0, - self.class.max_jobs)
+      end
+
+      true
+    end
+
+    # Yield block inside an existing multi connection or creates new one
+    def with_multi_connection(conn, &block)
+      return yield(conn) if conn
+
+      Sidekiq.redis do |c|
+        c.multi do |multi|
+          yield(multi)
+        end
+      end
+    end
+
+    def retry_all
+      each(&:retry) while size > 0
+    end
+
+    def self.max_jobs
+      Sidekiq.options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
+    end
+
+    def self.timeout
+      Sidekiq.options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
+    end
+  end
+end

data/spec/base_reliable_fetch_spec.rb

@@ -5,6 +5,8 @@ require 'sidekiq/reliable_fetch'
 require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::BaseReliableFetch do
+  let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
+
   before { Sidekiq.redis(&:flushdb) }
 
   describe 'UnitOfWork' do
@@ -12,7 +14,7 @@ describe Sidekiq::BaseReliableFetch do
 
     describe '#requeue' do
       it 'requeues job' do
-        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
 
         uow = fetcher.retrieve_work
 
@@ -25,7 +27,7 @@ describe Sidekiq::BaseReliableFetch do
 
     describe '#acknowledge' do
       it 'acknowledges job' do
-        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
 
         uow = fetcher.retrieve_work
 
@@ -38,19 +40,42 @@ describe Sidekiq::BaseReliableFetch do
   end
 
   describe '.bulk_requeue' do
+    let!(:queue1) { Sidekiq::Queue.new('foo') }
+    let!(:queue2) { Sidekiq::Queue.new('bar') }
+
     it 'requeues the bulk' do
-      queue1 = Sidekiq::Queue.new('foo')
-      queue2 = Sidekiq::Queue.new('bar')
+      uow = described_class::UnitOfWork
+      jobs = [ uow.new('queue:foo', job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
+      described_class.bulk_requeue(jobs, queues: [])
 
-      expect(queue1.size).to eq 0
-      expect(queue2.size).to eq 0
+      expect(queue1.size).to eq 2
+      expect(queue2.size).to eq 1
+    end
 
+    it 'puts jobs into interrupted queue' do
       uow = described_class::UnitOfWork
-      jobs = [ uow.new('queue:foo', 'bob'), uow.new('queue:foo', 'bar'), uow.new('queue:bar', 'widget') ]
+      interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
+      jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
+      described_class.bulk_requeue(jobs, queues: [])
+
+      expect(queue1.size).to eq 1
+      expect(queue2.size).to eq 1
+      expect(Sidekiq::InterruptedSet.new.size).to eq 1
+    end
+
+    it 'does not put jobs into interrupted queue if it is disabled' do
+      Sidekiq.options[:max_retries_after_interruption] = -1
+
+      uow = described_class::UnitOfWork
+      interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
+      jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
       described_class.bulk_requeue(jobs, queues: [])
 
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
+      expect(Sidekiq::InterruptedSet.new.size).to eq 0
+
+      Sidekiq.options[:max_retries_after_interruption] = 3
     end
   end
 

data/spec/fetch_shared_examples.rb

@@ -4,44 +4,65 @@ shared_examples 'a Sidekiq fetcher' do
   before { Sidekiq.redis(&:flushdb) }
 
   describe '#retrieve_work' do
+    let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
     let(:fetcher) { described_class.new(queues: ['assigned']) }
 
     it 'retrieves the job and puts it to working queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:assigned', 'msg') }
+      Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
 
       uow = fetcher.retrieve_work
 
       expect(working_queue_size('assigned')).to eq 1
       expect(uow.queue_name).to eq 'assigned'
-      expect(uow.job).to eq 'msg'
+      expect(uow.job).to eq job
       expect(Sidekiq::Queue.new('assigned').size).to eq 0
     end
 
     it 'does not retrieve a job from foreign queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', 'msg') }
+      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', job) }
 
       expect(fetcher.retrieve_work).to be_nil
     end
 
-    it 'requeues jobs from dead working queue' do
+    it 'requeues jobs from dead working queue with incremented interrupted_count' do
       Sidekiq.redis do |conn|
-        conn.rpush(other_process_working_queue_name('assigned'), 'msg')
+        conn.rpush(other_process_working_queue_name('assigned'), job)
       end
 
+      expected_job = Sidekiq.load_json(job)
+      expected_job['interrupted_count'] = 1
+      expected_job = Sidekiq.dump_json(expected_job)
+
       uow = fetcher.retrieve_work
 
-      expect(uow.job).to eq 'msg'
+      expect(uow.job).to eq expected_job
 
       Sidekiq.redis do |conn|
         expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
       end
     end
 
+    it 'ignores working queue keys in unknown formats' do
+      # Add a spurious non-numeric char segment at the end; this simulates any other
+      # incorrect form in general
+      malformed_key = "#{other_process_working_queue_name('assigned')}:X"
+      Sidekiq.redis do |conn|
+        conn.rpush(malformed_key, job)
+      end
+
+      uow = fetcher.retrieve_work
+
+      Sidekiq.redis do |conn|
+        expect(conn.llen(malformed_key)).to eq 1
+      end
+    end
+
+
     it 'does not requeue jobs from live working queue' do
       working_queue = live_other_process_working_queue_name('assigned')
 
       Sidekiq.redis do |conn|
-        conn.rpush(working_queue, 'msg')
+        conn.rpush(working_queue, job)
       end
 
       uow = fetcher.retrieve_work
@@ -56,8 +77,7 @@ shared_examples 'a Sidekiq fetcher' do
     it 'does not clean up orphaned jobs more than once per cleanup interval' do
       Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
 
-      expect_any_instance_of(described_class)
-        .to receive(:clean_working_queues!).once
+      expect(described_class).to receive(:clean_working_queues!).once
 
       threads = 10.times.map do
         Thread.new do

data/spec/reliable_fetch_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'fetch_shared_examples'
+require 'sidekiq/base_reliable_fetch'
 require 'sidekiq/reliable_fetch'
 
 describe Sidekiq::ReliableFetch do

data/spec/semi_reliable_fetch_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'fetch_shared_examples'
+require 'sidekiq/base_reliable_fetch'
 require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::SemiReliableFetch do

data/tests/README.md

@@ -0,0 +1,37 @@
+# How to run reliability tests
+
+```
+cd reliability_test
+bundle exec ruby reliability_test.rb
+```
+
+You can adjust some parameters of the test in the `config.rb`.
+
+JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
+
+You need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
+`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).
+
+
+## How it works
+
+This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
+number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
+
+# How to run interruption tests
+
+```
+cd tests/interruption
+
+# Verify "KILL" signal
+bundle exec ruby test_kill_signal.rb
+
+# Verify "TERM" signal
+bundle exec ruby test_term_signal.rb
+```
+
+It requires Redis to be running on 6379 port.
+
+## How it works
+
+It spawns Sidekiq workers then creates a job that will kill itself after a moment. The  reliable fetcher will bring it back. The purpose is to verify that job is run no more then allowed number of times.

data/tests/interruption/config.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative '../../lib/sidekiq-reliable-fetch'
+require_relative 'worker'
+
+TEST_CLEANUP_INTERVAL = 20
+TEST_LEASE_INTERVAL = 5
+
+Sidekiq.configure_server do |config|
+  config.options[:semi_reliable_fetch] = true
+
+  # We need to override these parameters to not wait too long
+  # The default values are good for production use only
+  # These will be ignored for :basic
+  config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+  config.options[:lease_interval] = TEST_LEASE_INTERVAL
+
+  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+end

data/tests/interruption/test_kill_signal.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require_relative 'config'
+require_relative '../support/utils'
+
+EXPECTED_NUM_TIMES_BEEN_RUN = 3
+NUM_WORKERS = EXPECTED_NUM_TIMES_BEEN_RUN + 1
+
+Sidekiq.redis(&:flushdb)
+
+pids = spawn_workers(NUM_WORKERS)
+
+RetryTestWorker.perform_async
+
+sleep 300
+
+Sidekiq.redis do |redis|
+  times_has_been_run = redis.get('times_has_been_run').to_i
+  assert 'The job has been run', times_has_been_run, EXPECTED_NUM_TIMES_BEEN_RUN
+end
+
+assert 'Found interruption exhausted jobs', Sidekiq::InterruptedSet.new.size, 1
+
+stop_workers(pids)

data/tests/interruption/test_term_signal.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require_relative 'config'
+require_relative '../support/utils'
+
+EXPECTED_NUM_TIMES_BEEN_RUN = 3
+NUM_WORKERS = EXPECTED_NUM_TIMES_BEEN_RUN + 1
+
+Sidekiq.redis(&:flushdb)
+
+pids = spawn_workers(NUM_WORKERS)
+
+RetryTestWorker.perform_async('TERM', 60)
+
+sleep 300
+
+Sidekiq.redis do |redis|
+  times_has_been_run = redis.get('times_has_been_run').to_i
+  assert 'The job has been run', times_has_been_run, EXPECTED_NUM_TIMES_BEEN_RUN
+end
+
+assert 'Found interruption exhausted jobs', Sidekiq::InterruptedSet.new.size, 1
+
+stop_workers(pids)

data/tests/interruption/worker.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class RetryTestWorker
+  include Sidekiq::Worker
+
+  def perform(signal = 'KILL', wait_seconds = 1)
+    Sidekiq.redis do |redis|
+      redis.incr('times_has_been_run')
+    end
+
+    Process.kill(signal, Process.pid)
+
+    sleep wait_seconds
+  end
+end

data/{test → tests/reliability}/config.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../lib/sidekiq/base_reliable_fetch'
-require_relative '../lib/sidekiq/reliable_fetch'
-require_relative '../lib/sidekiq/semi_reliable_fetch'
+require_relative '../../lib/sidekiq-reliable-fetch'
 require_relative 'worker'
 
 REDIS_FINISHED_LIST = 'reliable-fetcher-finished-jids'

data/{test → tests/reliability}/reliability_test.rb

@@ -89,7 +89,7 @@ Sidekiq.redis(&:flushdb)
 jobs = []
 
 NUMBER_OF_JOBS.times do
-  jobs << TestWorker.perform_async
+  jobs << ReliabilityTestWorker.perform_async
 end
 
 puts "Queued #{NUMBER_OF_JOBS} jobs"

data/{test → tests/reliability}/worker.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class TestWorker
+class ReliabilityTestWorker
   include Sidekiq::Worker
 
   def perform

data/tests/support/utils.rb

@@ -0,0 +1,26 @@
+def assert(text, actual, expected)
+  if actual == expected
+    puts "#{text}: #{actual} (Success)"
+  else
+    puts "#{text}: #{actual} (Failed). Expected: #{expected}"
+    exit 1
+  end
+end
+
+def spawn_workers(number)
+  pids = []
+
+  number.times do
+    pids << spawn('sidekiq -r ./config.rb')
+  end
+
+  pids
+end
+
+# Stop Sidekiq workers
+def stop_workers(pids)
+  pids.each do |pid|
+    Process.kill('KILL', pid)
+    Process.wait pid
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.3
 platform: ruby
 authors:
 - TEA
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 2021-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -42,6 +42,7 @@ files:
 - gitlab-sidekiq-fetcher.gemspec
 - lib/sidekiq-reliable-fetch.rb
 - lib/sidekiq/base_reliable_fetch.rb
+- lib/sidekiq/interrupted_set.rb
 - lib/sidekiq/reliable_fetch.rb
 - lib/sidekiq/semi_reliable_fetch.rb
 - spec/base_reliable_fetch_spec.rb
@@ -49,10 +50,15 @@ files:
 - spec/reliable_fetch_spec.rb
 - spec/semi_reliable_fetch_spec.rb
 - spec/spec_helper.rb
-- test/README.md
-- test/config.rb
-- test/reliability_test.rb
-- test/worker.rb
+- tests/README.md
+- tests/interruption/config.rb
+- tests/interruption/test_kill_signal.rb
+- tests/interruption/test_term_signal.rb
+- tests/interruption/worker.rb
+- tests/reliability/config.rb
+- tests/reliability/reliability_test.rb
+- tests/reliability/worker.rb
+- tests/support/utils.rb
 homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
@@ -72,8 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq

data/test/README.md

@@ -1,34 +0,0 @@
-# How to run
-
-```
-cd test
-bundle exec ruby reliability_test.rb
-```
-
-You can adjust some parameters of the test in the `config.rb`
-
-
-# How it works
-
-This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
-number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
-
-# How to run tests
-
-To run rspec:
-
-```
-bundle exec rspec
-```
-
-To run performance tests:
-
-```
-cd test
-JOB_FETCHER=semi bundle exec ruby reliability_test.rb
-```
-
-JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
-
-To run both kind of tests you need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
-`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).